From c99a7451a7c214c7e0388bc609a02b47fb54d156 Mon Sep 17 00:00:00 2001 From: Ravishankar Gnanaprakasam Date: Fri, 15 May 2026 12:06:07 +0530 Subject: [PATCH 1/2] [ci-cd] Add markdownlint and misspell [ci-cd] Add markdownlint and misspell [ci-cd] Add markdownlint and misspell --- .github/workflows/build-and-test.yml | 4 + .markdownlint.yaml | 35 +++ .markdownlintignore | 6 + CHANGELOG.md | 2 +- CONTRIBUTING.md | 13 +- Makefile | 18 ++ README.md | 4 +- cmd/opampsupervisor/specification/README.md | 4 +- cmd/schemagen/README.md | 6 +- confmap/provider/aesprovider/README.md | 6 +- .../googlesecretmanagerprovider/README.md | 3 +- connector/metricsaslogsconnector/README.md | 3 +- connector/otlpjsonconnector/README.md | 1 - connector/roundrobinconnector/README.md | 2 +- connector/servicegraphconnector/README.md | 2 +- connector/signaltometricsconnector/README.md | 6 +- connector/spanmetricsconnector/README.md | 16 +- docs/new-components.md | 6 +- docs/release.md | 1 - examples/kubernetes/dev-docs.md | 1 - examples/nomad/README.md | 1 - examples/secure-tracing/README.md | 21 +- exporter/alertmanagerexporter/README.md | 1 - .../alibabacloudlogserviceexporter/README.md | 1 - exporter/awscloudwatchlogsexporter/README.md | 2 - exporter/awsemfexporter/README.md | 22 +- exporter/awskinesisexporter/README.md | 2 +- exporter/awss3exporter/README.md | 6 +- exporter/awsxrayexporter/README.md | 4 +- exporter/azureblobexporter/README.md | 1 - exporter/azuredataexplorerexporter/README.md | 2 +- .../azuremonitorexporter/AUTHENTICATION.md | 2 - exporter/azuremonitorexporter/README.md | 2 +- exporter/clickhouseexporter/README.md | 2 - exporter/coralogixexporter/README.md | 9 +- exporter/datadogexporter/README.md | 1 - exporter/datasetexporter/README.md | 8 +- exporter/dorisexporter/README.md | 2 +- exporter/elasticsearchexporter/README.md | 18 +- .../elasticsearchexporter/documentation.md | 1 - exporter/faroexporter/README.md | 2 +- exporter/fileexporter/README.md | 12 +- exporter/googlecloudexporter/README.md | 12 +- exporter/googlecloudpubsubexporter/README.md | 40 +-- .../googlemanagedprometheusexporter/README.md | 2 +- exporter/honeycombmarkerexporter/README.md | 6 +- exporter/kafkaexporter/README.md | 2 +- exporter/logicmonitorexporter/README.md | 2 +- exporter/logzioexporter/README.md | 4 +- exporter/mezmoexporter/README.md | 2 +- exporter/opensearchexporter/README.md | 1 - .../prometheusremotewriteexporter/DESIGN.md | 77 ++---- .../prometheusremotewriteexporter/README.md | 13 +- exporter/pulsarexporter/README.md | 2 +- exporter/rabbitmqexporter/README.md | 2 +- exporter/sematextexporter/README.md | 2 +- exporter/signalfxexporter/README.md | 24 +- .../docs/translation_rules_migration_guide.md | 2 +- exporter/splunkhecexporter/README.md | 7 +- exporter/stefexporter/README.md | 26 +- exporter/tinybirdexporter/README.md | 3 - extension/ackextension/README.md | 1 - extension/asapauthextension/README.md | 4 +- extension/awsproxy/README.md | 2 - extension/basicauthextension/README.md | 2 +- extension/bearertokenauthextension/README.md | 1 - extension/encoding/README.md | 6 +- .../avrologencodingextension/README.md | 1 - .../README.md | 1 - .../awslogsencodingextension/README.md | 10 +- .../encoding/azureencodingextension/README.md | 1 - .../README.md | 48 ++-- .../jsonlogencodingextension/README.md | 1 - .../encoding/otlpencodingextension/README.md | 1 - .../skywalkingencodingextension/README.md | 1 - extension/headerssetterextension/README.md | 2 +- extension/healthcheckextension/README.md | 1 - extension/healthcheckv2extension/README.md | 1 - extension/httpforwarderextension/README.md | 1 - extension/jaegerremotesampling/README.md | 2 - extension/k8sleaderelector/README.md | 1 - .../testdata/README.md | 2 +- extension/observer/cfgardenobserver/README.md | 1 - extension/observer/dockerobserver/README.md | 3 +- extension/observer/ecsobserver/README.md | 34 +-- extension/observer/k8sobserver/README.md | 4 +- extension/opampcustommessages/README.md | 49 ++-- extension/opampextension/README.md | 2 +- extension/pprofextension/README.md | 2 - extension/sigv4authextension/README.md | 3 - extension/sigv4authextension/design.md | 235 ++++++------------ extension/storage/filestorage/README.md | 6 +- internal/otelarrow/admission2/README.md | 2 +- internal/otelarrow/netstats/README.md | 10 +- issue-triaging.md | 2 +- .../agentcomponents_options.md | 194 +++++++-------- pkg/expohisto/README.md | 20 +- pkg/golden/README.md | 26 +- pkg/ottl/LANGUAGE.md | 3 +- pkg/ottl/README.md | 1 - pkg/ottl/contexts/README.md | 2 +- pkg/ottl/contexts/ottldatapoint/README.md | 2 +- pkg/ottl/contexts/ottllog/README.md | 46 ++-- pkg/ottl/contexts/ottlotelcol/README.md | 10 +- pkg/ottl/contexts/ottlspan/README.md | 10 +- pkg/ottl/contexts/ottlspanevent/README.md | 12 +- pkg/ottl/ottlfuncs/README.md | 104 ++------ pkg/pdatatest/README.md | 78 +++--- pkg/stanza/docs/operators/README.md | 1 - pkg/stanza/docs/operators/add.md | 1 - pkg/stanza/docs/operators/container.md | 4 +- pkg/stanza/docs/operators/file_input.md | 2 +- pkg/stanza/docs/operators/file_output.md | 1 - .../docs/operators/json_array_parser.md | 5 +- pkg/stanza/docs/operators/json_parser.md | 2 - pkg/stanza/docs/operators/move.md | 1 - pkg/stanza/docs/operators/noop.md | 1 - pkg/stanza/docs/operators/regex_parser.md | 1 - pkg/stanza/docs/operators/regex_replace.md | 2 +- pkg/stanza/docs/operators/router.md | 1 - .../docs/operators/scope_name_parser.md | 1 - pkg/stanza/docs/operators/stdout.md | 1 - pkg/stanza/docs/operators/syslog_input.md | 4 - pkg/stanza/docs/operators/syslog_parser.md | 1 - pkg/stanza/docs/operators/tcp_input.md | 4 +- pkg/stanza/docs/operators/time_parser.md | 1 - pkg/stanza/docs/operators/trace_parser.md | 1 - pkg/stanza/docs/operators/udp_input.md | 4 +- pkg/stanza/docs/operators/uri_parser.md | 2 - .../docs/operators/windows_eventlog_input.md | 38 +-- pkg/stanza/docs/types/bytesize.md | 1 - pkg/stanza/docs/types/entry.md | 1 - pkg/stanza/docs/types/field.md | 1 - pkg/stanza/docs/types/scope_name.md | 2 - pkg/stanza/docs/types/severity.md | 1 - pkg/stanza/docs/types/timestamp.md | 1 - pkg/stanza/docs/types/trace.md | 2 - pkg/stanza/fileconsumer/design.md | 25 +- pkg/stanza/operator/input/file/design.md | 23 +- pkg/translator/azurelogs/README.md | 1 - pkg/xstreamencoding/README.md | 5 +- processor/attributesprocessor/README.md | 14 +- .../cardinalityguardianprocessor/README.md | 2 +- .../cumulativetodeltaprocessor/README.md | 1 - .../deltatocumulativeprocessor/README.md | 3 +- processor/deltatorateprocessor/README.md | 3 +- processor/filterprocessor/README.md | 6 +- processor/geoipprocessor/README.md | 1 - processor/groupbyattrsprocessor/README.md | 3 +- processor/groupbytraceprocessor/README.md | 10 +- processor/intervalprocessor/README.md | 2 +- processor/isolationforestprocessor/README.md | 6 +- processor/k8sattributesprocessor/README.md | 6 +- processor/logdedupprocessor/README.md | 3 +- processor/lookupprocessor/README.md | 1 - .../metricsgenerationprocessor/README.md | 2 +- processor/metricstarttimeprocessor/README.md | 1 - processor/metricstransformprocessor/README.md | 2 +- .../probabilisticsamplerprocessor/README.md | 2 +- processor/redactionprocessor/README.md | 2 - processor/remotetapprocessor/README.md | 2 +- .../resourcedetectionprocessor/README.md | 98 ++++---- processor/schemaprocessor/DESIGN.md | 4 +- processor/schemaprocessor/GUIDE.md | 2 +- processor/spanprocessor/README.md | 2 +- processor/tailsamplingprocessor/README.md | 2 +- processor/transformprocessor/README.md | 68 +++-- processor/unrollprocessor/README.md | 4 - receiver/activedirectorydsreceiver/README.md | 1 - receiver/aerospikereceiver/README.md | 2 - receiver/apachereceiver/README.md | 2 - receiver/apachesparkreceiver/README.md | 1 - receiver/awscloudwatchreceiver/README.md | 1 - .../awscontainerinsightreceiver/README.md | 76 +++--- .../awscontainerinsightreceiver/design.md | 16 +- .../awsecscontainermetricsreceiver/README.md | 111 ++++----- receiver/awsfirehosereceiver/README.md | 1 - receiver/awslambdareceiver/README.md | 20 +- receiver/awss3receiver/README.md | 1 - receiver/azureblobreceiver/README.md | 4 - receiver/azureeventhubreceiver/README.md | 1 - receiver/azuremonitorreceiver/README.md | 14 +- .../concurrency_bench_report.md | 1 - .../deepcopy_bench_report.md | 100 ++++---- receiver/carbonreceiver/README.md | 2 - receiver/ciscoosreceiver/README.md | 1 - receiver/cloudflarereceiver/README.md | 2 - receiver/cloudfoundryreceiver/README.md | 6 +- receiver/collectdreceiver/README.md | 2 - receiver/couchdbreceiver/README.md | 2 - receiver/datadogreceiver/README.md | 1 - receiver/dockerstatsreceiver/README.md | 2 - receiver/elasticsearchreceiver/README.md | 1 - receiver/envoyalsreceiver/README.md | 2 - receiver/expvarreceiver/README.md | 11 +- receiver/faroreceiver/README.md | 2 - receiver/filelogreceiver/README.md | 5 +- receiver/flinkmetricsreceiver/README.md | 2 - receiver/fluentforwardreceiver/README.md | 1 - receiver/githubreceiver/README.md | 3 +- .../internal/scraper/githubscraper/README.md | 3 +- receiver/gitlabreceiver/README.md | 2 +- .../googlecloudmonitoringreceiver/README.md | 1 - receiver/googlecloudpubsubreceiver/README.md | 22 +- .../documentation.md | 2 - receiver/googlecloudspannerreceiver/README.md | 1 - .../googlecloudspannerreceiver/cardinality.md | 1 - receiver/haproxyreceiver/README.md | 3 +- receiver/hostmetricsreceiver/README.md | 1 - receiver/httpcheckreceiver/README.md | 2 - receiver/huaweicloudcesreceiver/README.md | 19 +- receiver/icmpcheckreceiver/README.md | 3 +- receiver/iisreceiver/README.md | 1 - receiver/influxdbreceiver/README.md | 2 - receiver/jaegerreceiver/README.md | 2 - receiver/jmxreceiver/README.md | 1 - receiver/k8sclusterreceiver/README.md | 9 +- receiver/k8seventsreceiver/README.md | 1 - receiver/k8sobjectsreceiver/README.md | 1 - receiver/kafkametricsreceiver/README.md | 1 - receiver/kafkareceiver/README.md | 1 - receiver/kafkareceiver/documentation.md | 1 - receiver/kubeletstatsreceiver/README.md | 1 - receiver/libhoneyreceiver/README.md | 3 +- receiver/lokireceiver/README.md | 1 - .../macosunifiedloggingreceiver/README.md | 4 - receiver/memcachedreceiver/README.md | 2 - receiver/mongodbatlasreceiver/README.md | 5 +- receiver/mongodbreceiver/README.md | 5 +- receiver/mysqlreceiver/README.md | 3 +- receiver/mysqlreceiver/documentation.md | 2 - receiver/namedpipereceiver/README.md | 1 - receiver/netflowreceiver/README.md | 1 - receiver/nginxreceiver/README.md | 1 - receiver/nsxtreceiver/README.md | 2 - receiver/ntpreceiver/README.md | 4 +- receiver/oracledbreceiver/README.md | 2 - receiver/osqueryreceiver/README.md | 1 - receiver/otelarrowreceiver/README.md | 3 +- receiver/otelarrowreceiver/config.md | 3 +- receiver/otlpjsonfilereceiver/README.md | 1 - receiver/podmanreceiver/README.md | 25 +- receiver/postgresqlreceiver/README.md | 19 +- receiver/postgresqlreceiver/documentation.md | 3 - receiver/pprofreceiver/README.md | 1 - receiver/prometheusreceiver/DESIGN.md | 14 +- receiver/prometheusreceiver/README.md | 9 +- .../prometheusremotewritereceiver/README.md | 3 +- receiver/pulsarreceiver/README.md | 5 +- receiver/purefareceiver/README.md | 4 +- receiver/purefbreceiver/README.md | 4 +- receiver/rabbitmqreceiver/README.md | 1 - receiver/receivercreator/README.md | 8 +- receiver/redfishreceiver/README.md | 3 +- receiver/redisreceiver/README.md | 16 +- receiver/redisreceiver/config.md | 3 +- receiver/riakreceiver/README.md | 3 - receiver/saphanareceiver/README.md | 2 - receiver/signalfxreceiver/README.md | 2 +- receiver/simpleprometheusreceiver/README.md | 1 - receiver/skywalkingreceiver/README.md | 1 - receiver/snmpreceiver/README.md | 4 +- receiver/snowflakereceiver/README.md | 1 - receiver/solacereceiver/README.md | 2 - receiver/splunkenterprisereceiver/README.md | 10 +- receiver/splunkhecreceiver/README.md | 1 - receiver/sqlqueryreceiver/README.md | 1 - receiver/sqlserverreceiver/README.md | 6 +- .../sqlserverreceiver/logs-documentation.md | 1 - receiver/sshcheckreceiver/README.md | 5 +- receiver/statsdreceiver/README.md | 9 +- receiver/stefreceiver/README.md | 1 - receiver/syslogreceiver/README.md | 1 - receiver/systemdreceiver/documentation.md | 1 - receiver/tcpcheckreceiver/README.md | 1 - receiver/tcplogreceiver/README.md | 4 +- receiver/udplogreceiver/README.md | 4 +- receiver/vcenterreceiver/README.md | 3 +- receiver/vcrreceiver/README.md | 1 - receiver/wavefrontreceiver/README.md | 2 - receiver/webhookeventreceiver/README.md | 8 +- receiver/windowseventlogreceiver/README.md | 5 +- .../windowsperfcountersreceiver/README.md | 18 +- receiver/windowsservicereceiver/README.md | 1 - .../windowsservicereceiver/documentation.md | 1 - receiver/yanggrpcreceiver/README.md | 3 +- receiver/zipkinreceiver/README.md | 1 - receiver/zookeeperreceiver/README.md | 1 - scraper/zookeeperscraper/README.md | 1 - testbed/README.md | 75 +++--- 290 files changed, 1113 insertions(+), 1543 deletions(-) create mode 100644 .markdownlint.yaml create mode 100644 .markdownlintignore diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 73ef4ff3119a4..63f293fcfe5fb 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -132,6 +132,10 @@ jobs: - run: make genotelcontribcol - name: CheckDoc run: make checkdoc + - name: MarkdownLint + run: make markdownlint + - name: Misspell + run: make misspell - name: CheckMetadata run: make checkmetadata - name: CheckApi diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000000000..bd3df84e13631 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,35 @@ +# markdownlint configuration for OpenTelemetry Collector +# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md +# +# Start permissive to establish a baseline; tighten rules over time. +default: true + +# Disabled rules (existing codebase has many violations) +MD001: false # heading increment +MD004: false # unordered list style +MD007: false # unordered list indentation +MD013: false # line length +MD014: false # dollar signs in commands +MD022: false # blanks around headings +MD024: false # duplicate headings +MD025: false # multiple top-level headings +MD026: false # trailing punctuation in heading +MD029: false # ordered list prefix +MD030: false # spaces after list markers +MD031: false # blanks around fences +MD032: false # blanks around lists +MD033: false # inline HTML +MD034: false # bare URLs +MD036: false # emphasis as heading +MD038: false # spaces in code spans +MD040: false # fenced code language +MD041: false # first line heading +MD047: false # single trailing newline +MD049: false # emphasis style +MD051: false # link fragments +MD053: false # unused link definitions +MD058: false # blanks around tables +MD059: false # descriptive link text +MD060: false # table column style +MD028: false # blank line inside blockquote +MD037: false # spaces inside emphasis diff --git a/.markdownlintignore b/.markdownlintignore new file mode 100644 index 0000000000000..88ea04aff409c --- /dev/null +++ b/.markdownlintignore @@ -0,0 +1,6 @@ +# Changelogs (autogenerated, chloggen manages structure) +CHANGELOG.md +CHANGELOG-API.md + +# GitHub templates (intentional structure/HTML) +.github/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f9059189ecb..ebeac55f15d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,7 +180,7 @@ If you are looking for developer-facing changes, check out [CHANGELOG-API.md](./ Previously: - the `service.instance.id` reported in the AgentDescription was based on the OpAMP instance UID - the instance UID was typically set based on the `service.instance.id` from the Collector resource attributes - - it could be overriden using the `instance_uid` configuration of the OpAMP extension + - it could be overridden using the `instance_uid` configuration of the OpAMP extension This meant that the reported `service.instance.id` did not always match the Collector resource attributes, which is a problem for correlation, and that server implementations got used to the typical case of diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c101e7e55bc88..3090f000bb9e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,11 @@ Replace `otel-config.yaml` with the appropriate configuration file as needed. testing it against the provided configuration. 4. Lint your changes: + + - Markdown lint for entrie project. + ```shell + make markdownlint + ``` - For the entire project: ```shell @@ -113,7 +118,9 @@ No changelog entry: The title for your pull-request should contain the component type and name in brackets, plus a short statement for your change. For instance: - [processor/tailsampling] fix AND policy +``` +[processor/tailsampling] fix AND policy +``` Alternatively, if you have already written a changelog entry, you can set your PR title to `as per changelog` and a GitHub Action will automatically generate the PR title and description from your changelog entry YAML file(s). This @@ -232,8 +239,8 @@ Following these steps for contributing additional metrics to existing receivers. - To generate new metrics on top of this updated YAML file. - Run `cd receiver/redisreceiver` - Run `go generate ./...` -- Review the changed files and merge the changes into your forked repo. -- Create PR from Github web console following the instructions above. + - Review the changed files and merge the changes into your forked repo. + - Create PR from Github web console following the instructions above. ## General Recommendations Below are some recommendations that apply to typical components. These are not rigid rules and there are exceptions but diff --git a/Makefile b/Makefile index 4ff912059da9c..03f2925438259 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,13 @@ EX_INTERNAL=-not -path "./internal/*" EX_PKG=-not -path "./pkg/*" EX_CMD=-not -path "./cmd/*" +# All source code and documents. Used in spell check. +ALL_DOC := $(shell find . \( -name "*.md" -o -name "*.yaml" \) \ + -type f | sort) + +# All Markdown files. Used in markdownlint. +ALL_MD := $(shell find . -name "*.md" -type f | sort) + # This includes a final slash ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) @@ -601,6 +608,17 @@ build-examples: docker build -t otelcontribcol-fpm internal/buildscripts/packaging/fpm docker run --rm -v $(CURDIR):/repo -e PACKAGE=$* -e VERSION=$(VERSION) -e ARCH=$(ARCH) otelcontribcol-fpm + +# Runs lint for all markdown files +.PHONY: markdownlint +markdownlint: + npx -y markdownlint-cli@0.48.0 -c .markdownlint.yaml --ignore-path .markdownlintignore -- $(ALL_MD) + +# Fixes some of the markdown lint +.PHONY: markdownlintfix +markdownlintfix: + npx -y markdownlint-cli@0.48.0 -c .markdownlint.yaml --ignore-path .markdownlintignore --fix -- $(ALL_MD) + # Verify existence of READMEs for components specified as default components in the collector. .PHONY: checkdoc checkdoc: diff --git a/README.md b/README.md index 62ae27409d4d8..bd37b67064d0c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ # OpenTelemetry Collector Contrib -This is a repository for OpenTelemetry Collector components that are not suitable for the [core repository](https://github.com/open-telemetry/opentelemetry-collector) of the collector. +This is a repository for OpenTelemetry Collector components that are not suitable for the [core repository](https://github.com/open-telemetry/opentelemetry-collector) of the collector. The official distributions, core and contrib, are available as part of the [opentelemetry-collector-releases](https://github.com/open-telemetry/opentelemetry-collector-releases) repository. Some of the components in this repository are part of the "core" distribution, such as the Jaeger and Prometheus components, but most of the components here are only available as part of the "contrib" distribution. Users of the OpenTelemetry Collector are also encouraged to build their own custom distributions with the [OpenTelemetry Collector Builder](https://github.com/open-telemetry/opentelemetry-collector/tree/main/cmd/builder), using the components they need from the core repository, the contrib repository, and possibly third-party or internal repositories. @@ -160,4 +160,4 @@ behavior. The facilitators will typically rely on codeowner's detailed review of when making the final approval decision. Marking the PR with the `ready to merge` label should only happen (by triagers/approvers/maintainers) -once there is at least one approval from an approver, as per the description above. +once there is at least one approval from an approver, as per the description above. diff --git a/cmd/opampsupervisor/specification/README.md b/cmd/opampsupervisor/specification/README.md index 97b3ce5b93b79..6d84f3c06a33f 100644 --- a/cmd/opampsupervisor/specification/README.md +++ b/cmd/opampsupervisor/specification/README.md @@ -317,9 +317,9 @@ This results in the following Collector process invocation: If `extensions` are configured but the gate is disabled, the Supervisor will not start and the error message names the gate to enable. -These are instances of extensions specific to the Supervisor and are +These are instances of extensions specific to the Supervisor and are distinct from any extensions configured to run inside the managed -Collector configuration. Supervisor extensions are loaded and managed by the +Collector configuration. Supervisor extensions are loaded and managed by the Supervisor process itself. Supervisor extensions are configured under a top-level `extensions:` key diff --git a/cmd/schemagen/README.md b/cmd/schemagen/README.md index 1ad55cdaa250f..da78b98947724 100644 --- a/cmd/schemagen/README.md +++ b/cmd/schemagen/README.md @@ -11,13 +11,13 @@ and the generation process would be reversed – Go config files from schemas. ## Modes -Script can generate schemas for components and packages. There are two distinct modes of operation: +Script can generate schemas for components and packages. There are two distinct modes of operation: `component` and `package`. Schemagen automatically detects the mode based on metadata.yaml file content. ### Component mode The `component` mode is aimed at generating configurations for individual components like receivers, processors, -exporters, and connectors. +exporters, and connectors. In component mode, schema is generated in file named `config.schema.` to the chosen output folder (by default it's input directory). You can optionally specify the root struct to use for schema generation @@ -83,7 +83,7 @@ componentOverrides: configName: 'FileLogConfig' ``` -- `namespace` corresponds to the Go package import path for modules from current repository. +- `namespace` corresponds to the Go package import path for modules from current repository. It is used to resolve references to other types within the same repository. - `mappings` tell schemagen how to treat specific selector expressions as primitive schema fields. Each mapping converts the Go type into a scalar diff --git a/confmap/provider/aesprovider/README.md b/confmap/provider/aesprovider/README.md index 83bcd6ab6a528..9443c2aa9cba6 100644 --- a/confmap/provider/aesprovider/README.md +++ b/confmap/provider/aesprovider/README.md @@ -28,15 +28,15 @@ openssl rand -base64 32 Use placeholders with the following pattern `${aes:}` in a configuration. The value will be decrypted using the AES key provided in the environment variable `OTEL_AES_CREDENTIAL_PROVIDER` > For example: -> +> > ```shell > export OTEL_AES_CREDENTIAL_PROVIDER="GQi+Y8HwOYzs8lAOjHUqB7vXlN8bVU2k0TAKtzwJzac=" > ``` -> +> > ```yaml > password: ${aes:RsEf6cTWrssi8tlssfs1AJs2bRMrVm2Ce5TaWPY=} > ``` -> +> > will resolve to: > ```yaml > password: '1' diff --git a/confmap/provider/googlesecretmanagerprovider/README.md b/confmap/provider/googlesecretmanagerprovider/README.md index 579120bb25f0c..d5bb6e3b55feb 100644 --- a/confmap/provider/googlesecretmanagerprovider/README.md +++ b/confmap/provider/googlesecretmanagerprovider/README.md @@ -51,7 +51,6 @@ service: ### Prerequisites 1. Make sure to enable access to the [Secret Manager API](https://cloud.google.com/secret-manager/docs/accessing-the-api). -2. Make sure to [add the secret entries to Google Secret Manager](https://cloud.google.com/secret-manager/docs/create-secret-quickstart) before referencing them in the collector configurations. +2. Make sure to [add the secret entries to Google Secret Manager](https://cloud.google.com/secret-manager/docs/create-secret-quickstart) before referencing them in the collector configurations. 3. This Provider interacts with Google Secret Manager using the Secret Manager client library. This library uses [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials) to locate authentication credentials for Secret Manager. Therefore, if you run your collector in a local environment, execute the [`gcloud auth application-default login`](https://cloud.google.com/secret-manager/docs/authentication#client-libs) command to generate the necessary credential file to provide to ADC. 4. However, if your collector runs on Google Compute Engine (GCE) or Google Kubernetes Engine (GKE), running `gcloud auth application-default login` is optional. This is because ADC can retrieve credentials via [the metadata server](https://cloud.google.com/docs/authentication/application-default-credentials#order). However, ensure that your GKE or GCE instance [has enabled the cloud-platform OAuth scope](https://cloud.google.com/secret-manager/docs/accessing-the-api#oauth-scopes). Additionally, verify that the Service Account attached to the GCE or GKE instance has been granted at least the [roles/secretmanager.secretAccessor](https://cloud.google.com/secret-manager/docs/access-control#secret-manager-roles) IAM role to access secret entries in Google Secret Manager. - diff --git a/connector/metricsaslogsconnector/README.md b/connector/metricsaslogsconnector/README.md index 158a5541ad9a8..910e8dae6d79b 100644 --- a/connector/metricsaslogsconnector/README.md +++ b/connector/metricsaslogsconnector/README.md @@ -82,7 +82,6 @@ connectors: include_scope_info: false ``` - ### Example Metric Conversions For a gauge metric `cpu_usage` with value `85.2`: @@ -122,7 +121,7 @@ Each metric data point is converted to a log record with: - **Body**: Fixed JSON format: `{"metric_name": "$NAME", "value": "$VALUE"}` - **Timestamp**: Metric data point timestamp - **Observed Timestamp**: Metric data point start timestamp (if available) -- **Attributes**: +- **Attributes**: - Original metric data point attributes (labels) - `metric.name`: The metric name - `metric.type`: The metric type (Gauge, Sum, Histogram, etc.) diff --git a/connector/otlpjsonconnector/README.md b/connector/otlpjsonconnector/README.md index 5c23c25b1e5f1..931fa85436d25 100644 --- a/connector/otlpjsonconnector/README.md +++ b/connector/otlpjsonconnector/README.md @@ -30,7 +30,6 @@ > The old name `otlpjson` still works but is deprecated and will be removed in a future release. > Please update your configuration to use `otlp_json`. - Allows to extract otlpjson data from incoming Logs and specifically the `Body` field. The data is written in [Protobuf JSON diff --git a/connector/roundrobinconnector/README.md b/connector/roundrobinconnector/README.md index 0638a9e6ea325..a0ae079acbc1a 100644 --- a/connector/roundrobinconnector/README.md +++ b/connector/roundrobinconnector/README.md @@ -47,7 +47,7 @@ connectors: round_robin: ``` -Preprocess data, then export using multiple exporter instances to scale the throughput if the exporter +Preprocess data, then export using multiple exporter instances to scale the throughput if the exporter does not support scale well (e.g. prometheusremotewrite). ```yaml diff --git a/connector/servicegraphconnector/README.md b/connector/servicegraphconnector/README.md index f2c97016b602d..a187b08d44b97 100644 --- a/connector/servicegraphconnector/README.md +++ b/connector/servicegraphconnector/README.md @@ -125,7 +125,7 @@ The following settings are required: - `latency_histogram_buckets`: the list of durations defining the latency histogram buckets. Make sure use either `latency_histogram_buckets` or `exponential_histogram_max_size`. - Default: `[2ms, 4ms, 6ms, 8ms, 10ms, 50ms, 100ms, 200ms, 400ms, 800ms, 1s, 1400ms, 2s, 5s, 10s, 15s]` -- `exponential_histogram_max_size`: (no default) the maximum number of buckets per positive or negative number range. +- `exponential_histogram_max_size`: (no default) the maximum number of buckets per positive or negative number range. - `dimensions`: the list of dimensions to add together with the default dimensions defined above. The following settings can be optionally configured: diff --git a/connector/signaltometricsconnector/README.md b/connector/signaltometricsconnector/README.md index 8d06acc4d272a..02df6c30b4926 100644 --- a/connector/signaltometricsconnector/README.md +++ b/connector/signaltometricsconnector/README.md @@ -150,10 +150,10 @@ gauge: value: ``` -- [**Required**] `value`represents an OTTL expression to extract a numeric value from - the signal. Only OTTL expressions that return a value are accepted. The returned +- [**Required**] `value`represents an OTTL expression to extract a numeric value from + the signal. Only OTTL expressions that return a value are accepted. The returned value determines the value type of the `gauge` metric (`int` or `double`). - - For logs: Use e.g. `ExtractGrokPatterns` with a single key selector (see below). + - For logs: Use e.g. `ExtractGrokPatterns` with a single key selector (see below). - For other signals: Use a field such as `value_int`, `value_double`, or a valid OTTL expression. **Examples:** diff --git a/connector/spanmetricsconnector/README.md b/connector/spanmetricsconnector/README.md index e8429d1277be8..dcfae96ae45a6 100644 --- a/connector/spanmetricsconnector/README.md +++ b/connector/spanmetricsconnector/README.md @@ -27,7 +27,7 @@ > The old name `spanmetrics` still works but is deprecated and will be removed in a future release. > Please update your configuration to use `span_metrics`. -⚠️ Breaking Change Warning: +⚠️ Breaking Change Warning: The default duration metrics unit will change from `ms` to `s` to adhere to the OpenTelemetry semantic conventions and a feature gate `connector.spanmetrics.useSecondAsDefaultMetricsUnit` is also added. Currently, the feature gate is disabled by default, so the unit will remain `ms`. After one release cycle, the unit will switch to `s` and the feature gate will also be enabled by default. @@ -115,7 +115,7 @@ The following settings can be optionally configured: - `disable` (default: `false`): Disable all histogram metrics. - `unit` (default: `ms`): The time unit for recording duration measurements. calculated from spans duration measurements. One of either: `ms` or `s`. - - `dimensions`: additional attributes to add as dimensions to the `traces.span.metrics.duration` metric, + - `dimensions`: additional attributes to add as dimensions to the `traces.span.metrics.duration` metric, which will be included _on top of_ the common and configured `dimensions` for span attributes and resource attributes. - `explicit`: - `buckets`: the list of durations defining the duration histogram time buckets. Default @@ -129,14 +129,14 @@ The following settings can be optionally configured: If the `name`d attribute is missing in the span, the optional provided `default` is used. If no `default` is provided, this dimension will be **omitted** from the metric. -- `calls_dimensions`: additional attributes to add as dimensions to the `traces.span.metrics.calls` metric, +- `calls_dimensions`: additional attributes to add as dimensions to the `traces.span.metrics.calls` metric, which will be included _on top of_ the common and configured `dimensions` for span attributes and resource attributes. -- `exclude_dimensions`: the list of dimensions to be excluded from the default set of dimensions. Use to exclude unneeded data from metrics. +- `exclude_dimensions`: the list of dimensions to be excluded from the default set of dimensions. Use to exclude unneeded data from metrics. - `dimensions_cache_size`: this setting is deprecated, please use aggregation_cardinality_limit instead. - `include_instrumentation_scope`: a list of instrumentation scope names to include from the traces. - `resource_metrics_cache_size` (default: `1000`): the size of the cache holding metrics for a service. This is mostly relevant for cumulative temporality to avoid memory leaks and correct metric timestamp resets. -- `aggregation_temporality` (default: `AGGREGATION_TEMPORALITY_CUMULATIVE`): Defines the aggregation temporality of the generated metrics. +- `aggregation_temporality` (default: `AGGREGATION_TEMPORALITY_CUMULATIVE`): Defines the aggregation temporality of the generated metrics. One of either `AGGREGATION_TEMPORALITY_CUMULATIVE` or `AGGREGATION_TEMPORALITY_DELTA`. - `namespace` (default: `traces.span.metrics`): Defines the namespace of the generated metrics. If `namespace` provided, generated metric name will be added `namespace.` prefix. - `metrics_flush_interval` (default: `60s`): Defines the flush interval of the generated metrics. @@ -274,8 +274,8 @@ For more example configuration covering various other use cases, please visit th ## Known Limitation: the Single Writer Principle Proper configuration of the `spanmetricsconnector` ensures compliance with the [Single Writer Principle](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#single-writer), -which is a core requirement in the OpenTelemetry metrics data model. Misconfiguration, however, -may allow multiple components to write to the same metric stream, resulting in data inconsistency, +which is a core requirement in the OpenTelemetry metrics data model. Misconfiguration, however, +may allow multiple components to write to the same metric stream, resulting in data inconsistency, metric conflicts, or the dropping of time series by metric backends. ### Why this happens @@ -448,7 +448,7 @@ Custom web frameworks are a common source of high cardinality span names. While **Example: Custom Web Framework in Java** -Consider a custom web framework that intercepts the generic route `/my-web-fwk/*` and dispatches requests like `/my-web-fwk/product/123456ABCD` or `/my-web-fwk/user/john.doe`. +Consider a custom web framework that intercepts the generic route `/my-web-fwk/*` and dispatches requests like `/my-web-fwk/product/123456ABCD` or `/my-web-fwk/user/john.doe`. The default Java Servlet instrumentation produces vague span names (`GET /my-web-fwk/*`), while directly using request URIs creates high cardinality (`GET /my-web-fwk/product/123456ABCD`). diff --git a/docs/new-components.md b/docs/new-components.md index c4bf291f73c00..51d6a198e568b 100644 --- a/docs/new-components.md +++ b/docs/new-components.md @@ -32,7 +32,7 @@ understand how you may leverage existing components for your use case. Components refer to connectors, exporters, extensions, processors, and receivers. As a first step, we require that you build your component outside of the [opentelemetry-collector-contrib repository](https://github.com/open-telemetry/opentelemetry-collector-contrib). This is also the -fastest way to start using your component and to publish it for others to consume if you want to. +fastest way to start using your component and to publish it for others to consume if you want to. A component is a Go module (library) built using the `go.opentelemetry.io/collector` set of libraries. These libraries contain examples (e.g. see the example on the @@ -182,7 +182,7 @@ status: package fooreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fooreceiver" ``` - Type `make generate`. This will trigger the [metadata generator](https://github.com/open-telemetry/opentelemetry-collector/blob/main/cmd/mdatagen/README.md#using-the-metadata-generator) to generate the associated code/documentation. -- Type `make gencodeowners`. This will trigger the regeneration of the `.github/CODEOWNERS` file. +- Type `make gencodeowners`. This will trigger the regeneration of the `.github/CODEOWNERS` file. When donating a component to the community, break it down into separate PRs as follows: @@ -220,7 +220,7 @@ When donating a component to the community, break it down into separate PRs as f * Please also run: - `make generate` - `make genotelcontribcol` - + * The component must be enabled only after sufficient testing and only when it meets [`Alpha` stability requirements](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#alpha). * Once your component has reached `Alpha` stability, you may also submit a PR to the [OpenTelemetry Collector Releases](https://github.com/open-telemetry/opentelemetry-collector-releases) repository to include your component in future releases of the OpenTelemetry Collector `contrib` distribution. * Once a new component has been added to the executable: diff --git a/docs/release.md b/docs/release.md index 49dec4e8ebeb9..8ce891b3a7441 100644 --- a/docs/release.md +++ b/docs/release.md @@ -68,5 +68,4 @@ process. See the [opentelemetry-collector release procedure][1] document for the release schedule and release manager rotation. - [1]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/release.md diff --git a/examples/kubernetes/dev-docs.md b/examples/kubernetes/dev-docs.md index 9c08dec1b9ac1..e1c2ec2ee65b2 100644 --- a/examples/kubernetes/dev-docs.md +++ b/examples/kubernetes/dev-docs.md @@ -3,7 +3,6 @@ Developing Collector's features that target to run on a Kubernetes environment require some special handling for building and running the patches locally. - In order to build the Collector from source and deploy it on a local kind cluster (a kind cluster is required to be installed already) use the following Makefile targets: diff --git a/examples/nomad/README.md b/examples/nomad/README.md index 6b23637c2c75e..bb2a0f63bf44b 100644 --- a/examples/nomad/README.md +++ b/examples/nomad/README.md @@ -6,7 +6,6 @@ For a detailed, step-by-step tutorial on how to run the files included in this d The tutorial includes instructions on how to set up a local Nomad environment using [HashiQube](https://github.com/avillela/hashiqube) (a fork of [servian/hashiqube](https://github.com/servian/hashiqube)). It spins up a Vagrant VM running [Nomad](https://nomadproject.io), [Consul](https://www.consul.io), [Vault](https://www.vaultproject.io) and [Traefik](https://traefik.io). Detailed instructions can be found in the project [readme](https://github.com/avillela/hashiqube#readme). - ## Additional Resources Check out the [OpenTelemetry Collector Nomad Pack](https://github.com/hashicorp/nomad-pack-community-registry/tree/main/packs/opentelemetry_collector) in the [Nomad Pack Community Registry](https://github.com/hashicorp/nomad-pack-community-registry). \ No newline at end of file diff --git a/examples/secure-tracing/README.md b/examples/secure-tracing/README.md index 57fc214c70728..42b081b7176aa 100644 --- a/examples/secure-tracing/README.md +++ b/examples/secure-tracing/README.md @@ -1,16 +1,15 @@ -# Build A Secure Trace Collection Pipeline +# Build A Secure Trace Collection Pipeline Implementing robust security measures is essential for any tracing ingestion service and infrastructure. This is an illustrative example of a secure setup encompassing the following features for trace ingestion: - Data Encryption with OTLP receiver supporting TLS. Transport Layer Security (TLS) is employed to encrypt traces in transit between the application and the OpenTelemetry (OTLP) endpoint, fortifying data security. -- Client Authentication via [Envoy](https://www.envoyproxy.io/docs/envoy/latest/start/start), a high-performance proxy. +- Client Authentication via [Envoy](https://www.envoyproxy.io/docs/envoy/latest/start/start), a high-performance proxy. Even though we can configure OTLP receiver with mTLS for client authentication, authorization is not supported by OpenTelemetry Collector. This is one of the reasons that we use Envoy for client authentication. It allows us to easily add authorization, ensuring that only authorized clients can submit traces to the ingestion service. -In this example, we also include a test via telementrygen: a tool provided from this repository for generating synthetic telemetry data, which helps verify the security features and overall functionality of the set up. - +In this example, we also include a test via telementrygen: a tool provided from this repository for generating synthetic telemetry data, which helps verify the security features and overall functionality of the set up. ## Data Encryption via TLS -The OpenTelemetry Collector has detailed [documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) on how to configure TLS. In this example, we enable TLS for receivers which leverages server configuration. +The OpenTelemetry Collector has detailed [documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) on how to configure TLS. In this example, we enable TLS for receivers which leverages server configuration. Example ``` @@ -53,16 +52,16 @@ typed_config: ## Setup Environment ### Generate Certificates -To generate various self-signed certificates, including those for Envoy and the OpenTelemetry Collector receiver, as well as tracing client certificate, we utilize the widely renowned open-source tool [OpenSSL](https://www.openssl.org/source/), OpenSSL 3.1.0 14 was tested. +To generate various self-signed certificates, including those for Envoy and the OpenTelemetry Collector receiver, as well as tracing client certificate, we utilize the widely renowned open-source tool [OpenSSL](https://www.openssl.org/source/), OpenSSL 3.1.0 14 was tested. -In the `certs` folder, you can find a set of `.ext` files which define the properties for a certificate. A `MakeFile` is provided to facilate the process. +In the `certs` folder, you can find a set of `.ext` files which define the properties for a certificate. A `MakeFile` is provided to facilate the process. ``` $ cd certs $ make clean && make all ``` ### Bring up services -We use docker compose to bring up Envoy and OpenTelemetry Collection Pipeline. Make sure the current folder is `secure-tracing`, +We use docker compose to bring up Envoy and OpenTelemetry Collection Pipeline. Make sure the current folder is `secure-tracing`, ``` $ docker compose up @@ -72,12 +71,12 @@ From the console window, verify that `Envoy` and `Collector` are up and running. ``` secure-tracing-otel-collector-1 | Error: cannot start pipelines: failed to load TLS config: failed to load TLS cert and key: read /etc/otel-collector.crt: is a directory ``` -It's most likely due to missing certificates. Follow the steps from section above to generate certificates. +It's most likely due to missing certificates. Follow the steps from section above to generate certificates. ## Run test ### Compile telemetrygen -From the root of this repository, +From the root of this repository, ``` $ cd cmd/telemetrygen $ go build . @@ -110,7 +109,7 @@ secure-tracing-otel-collector-1 | -> net.sock.peer.addr: Str(1.2.3.4) secure-tracing-otel-collector-1 | -> peer.service: Str(telemetrygen-client) ... -``` +``` ## Shutdown Services ``` $ docker compose down diff --git a/exporter/alertmanagerexporter/README.md b/exporter/alertmanagerexporter/README.md index f2c8fca9095fe..2b3b23e3be70a 100644 --- a/exporter/alertmanagerexporter/README.md +++ b/exporter/alertmanagerexporter/README.md @@ -22,7 +22,6 @@ The following settings are required: - `endpoint`: Alertmanager endpoint to send events - `severity`: Default severity for Alerts - The following settings are optional: - `timeout` `sending_queue` and `retry_on_failure` settings as provided by [Exporter Helper](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/exporterhelper#configuration). diff --git a/exporter/alibabacloudlogserviceexporter/README.md b/exporter/alibabacloudlogserviceexporter/README.md index 39106b335d0bf..16dac12d9d914 100644 --- a/exporter/alibabacloudlogserviceexporter/README.md +++ b/exporter/alibabacloudlogserviceexporter/README.md @@ -47,7 +47,6 @@ service: exporters: [alibabacloud_logservice] ``` - ## All Telemetry Data If you are using OpenTelemetry Collector to collect different types of telemetry data, you should send to different LogService's store. diff --git a/exporter/awscloudwatchlogsexporter/README.md b/exporter/awscloudwatchlogsexporter/README.md index cb11b56dd10a2..857ad9e527b91 100644 --- a/exporter/awscloudwatchlogsexporter/README.md +++ b/exporter/awscloudwatchlogsexporter/README.md @@ -22,7 +22,6 @@ NOTE: OpenTelemetry Logging support is experimental, hence this exporter is subj The following settings are required: - - `log_group_name`: The group name of the CloudWatch Logs. If it does not exist it will be created automatically. It supports several placeholder names. One valid example is `/aws/metrics/{ClusterName}`. It will search for ClusterName (or aws.ecs.cluster.name) resource attribute in the metrics data and replace with the actual cluster name. If none of them are found in the resource attribute map, {ClusterName} will be replaced by undefined. Similar way, for the {TaskId}, it searches for TaskId (or aws.ecs.task.id) key in the resource attribute map. For {NodeName}, it searches for NodeName (or k8s.node.name) - List of valid placeholders: - `{ClusterName}`: `aws.ecs.cluster.name` @@ -37,7 +36,6 @@ The following settings are required: - `{FaasVersion}`: `faas.version` - `log_stream_name`: The stream name of the CloudWatch Logs. If it does not exist it will be created automatically. It supports the same placeholders as `log_group_name` - The following settings can be optionally configured: - `region`: The AWS region where the log stream is in. Region must be specified if it is not already set in the default credential chain. diff --git a/exporter/awsemfexporter/README.md b/exporter/awsemfexporter/README.md index 49375e057b54b..12676e895fa1f 100644 --- a/exporter/awsemfexporter/README.md +++ b/exporter/awsemfexporter/README.md @@ -13,14 +13,14 @@ [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -This exporter converts OpenTelemetry metrics to +This exporter converts OpenTelemetry metrics to [AWS CloudWatch Embedded Metric Format(EMF)](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Embedded_Metric_Format_Specification.html) -and then sends them directly to CloudWatch Logs using the +and then sends them directly to CloudWatch Logs using the [PutLogEvents](https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html) API. ## Data Conversion -Convert OpenTelemetry ```Int64DataPoints```, ```DoubleDataPoints```, ```SummaryDataPoints``` metrics datapoints into -CloudWatch ```EMF``` structured log formats and send it to CloudWatch. Logs and Metrics will be displayed in +Convert OpenTelemetry ```Int64DataPoints```, ```DoubleDataPoints```, ```SummaryDataPoints``` metrics datapoints into +CloudWatch ```EMF``` structured log formats and send it to CloudWatch. Logs and Metrics will be displayed in CloudWatch console. NaN, Inf values are not supported by CloudWatch EMF and will be dropped by the exporter. ## Exporter Configuration @@ -41,10 +41,10 @@ The following exporter configuration parameters are supported. | `role_arn` | IAM role to upload segments to a different account. | | | `external_id` | Shared identitier used when assuming an IAM role in an external AWS account. | | | `max_retries` | Maximum number of retries before abandoning an attempt to post data. | 1 | -| `dimension_rollup_option` | DimensionRollupOption is the option for metrics dimension rollup. Three options are available: `NoDimensionRollup`, `SingleDimensionRollupOnly` and `ZeroAndSingleDimensionRollup`. The default value is `ZeroAndSingleDimensionRollup`. Enabling feature gate `awsemf.nodimrollupdefault` will set default to `NoDimensionRollup`. |"ZeroAndSingleDimensionRollup" (Enable both zero dimension rollup and single dimension rollup)| -| `resource_to_telemetry_conversion` | "resource_to_telemetry_conversion" is the option for converting resource attributes to telemetry attributes. It has only one config option- `enabled`. For metrics, if `enabled=true`, all the resource attributes will be converted to metric labels by default. See `Resource Attributes to Metric Labels` section below for examples. | `enabled=false` | -| `output_destination` | "output_destination" is an option to specify the EMFExporter output. Currently, two options are available. "cloudwatch" or "stdout" | `cloudwatch` | -| `detailed_metrics` | Retain detailed datapoint values in exported metrics (e.g instead of exporting a quantile as a statistical value, preserve the quantile's population) | `false` | +| `dimension_rollup_option` | DimensionRollupOption is the option for metrics dimension rollup. Three options are available: `NoDimensionRollup`, `SingleDimensionRollupOnly` and `ZeroAndSingleDimensionRollup`. The default value is `ZeroAndSingleDimensionRollup`. Enabling feature gate `awsemf.nodimrollupdefault` will set default to `NoDimensionRollup`. |"ZeroAndSingleDimensionRollup" (Enable both zero dimension rollup and single dimension rollup)| +| `resource_to_telemetry_conversion` | "resource_to_telemetry_conversion" is the option for converting resource attributes to telemetry attributes. It has only one config option- `enabled`. For metrics, if `enabled=true`, all the resource attributes will be converted to metric labels by default. See `Resource Attributes to Metric Labels` section below for examples. | `enabled=false` | +| `output_destination` | "output_destination" is an option to specify the EMFExporter output. Currently, two options are available. "cloudwatch" or "stdout" | `cloudwatch` | +| `detailed_metrics` | Retain detailed datapoint values in exported metrics (e.g instead of exporting a quantile as a statistical value, preserve the quantile's population) | `false` | | `parse_json_encoded_attr_values` | List of attribute keys whose corresponding values are JSON-encoded strings and will be converted to JSON structures in emf logs. For example, the attribute string value "{\\"x\\":5,\\"y\\":6}" will be converted to a json object: ```{"x": 5, "y": 6}``` | [ ] | | [`metric_declarations`](#metric_declaration) | List of rules for filtering exported metrics and their dimensions. | [ ] | | [`metric_descriptors`](#metric_descriptor) | List of rules for inserting or updating metric descriptors. | [ ] | @@ -77,13 +77,12 @@ A metric descriptor section allows the schema of a metric to be overwritten befo | `unit` | The overwritten value of unit. The [MetricDatum](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) contains a full list of supported unit values. | | | `overwrite` | `true` if the schema should be overwritten with the given specification, otherwise it will only be configured if empty. | false | - ## AWS Credential Configuration -This exporter follows default credential resolution for the +This exporter follows default credential resolution for the [aws-sdk-go](https://docs.aws.amazon.com/sdk-for-go/api/index.html). -Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the +Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the credential configuration. ## Metric Attributes @@ -97,7 +96,6 @@ The AWS EMF Exporter will interpret the following metric attributes to change ho ## Configuration Examples - ### Resource Attributes to Metric Labels `resource_to_telemetry_conversion` option can be enabled to convert all the resource attributes to metric labels. By default, this option is disabled. Users need to set `enabled=true` to opt-in. See the config example below. diff --git a/exporter/awskinesisexporter/README.md b/exporter/awskinesisexporter/README.md index ed52a99132551..1b15ba2f4cae9 100644 --- a/exporter/awskinesisexporter/README.md +++ b/exporter/awskinesisexporter/README.md @@ -28,7 +28,7 @@ The following settings can be optionally configured: - `role` (no default): The role to be used in order to send data to the kinesis stream - `encoding` - `name` (default = otlp): defines the export type to be used to send to kinesis (available is `otlp_proto`, `otlp_json`, `zipkin_proto`, `zipkin_json`, `jaeger_proto`) - - **Note** : `otlp_json` is considered experimental and _should not_ be used for production environments. + - **Note** : `otlp_json` is considered experimental and _should not_ be used for production environments. - `compression` (default = none): allows to set the compression type (defaults BestSpeed for all) before forwarding to kinesis (available is `flate`, `gzip`, `zlib` or `none`) - `max_records_per_batch` (default = 500, PutRecords limit): The number of records that can be batched together then sent to kinesis. - `max_record_size` (default = 1Mb, PutRecord(s) limit on record size): The max allowed size that can be exported to kinesis diff --git a/exporter/awss3exporter/README.md b/exporter/awss3exporter/README.md index 947930e252cec..4215a3db7b070 100644 --- a/exporter/awss3exporter/README.md +++ b/exporter/awss3exporter/README.md @@ -87,11 +87,11 @@ See https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/ ### resource_attrs_to_s3 - `s3_bucket`: Defines which resource attribute's value should be used as the S3 bucket. - When this option is set, it dynamically overrides `s3uploader/s3_bucket`. + When this option is set, it dynamically overrides `s3uploader/s3_bucket`. If the specified resource attribute exists in the data, its value will be used as the bucket; otherwise, `s3uploader/s3_bucket` will serve as the fallback. - `s3_prefix`: Defines which resource attribute's value should be used as the S3 prefix. - When this option is set, it dynamically overrides `s3uploader/s3_prefix`. + When this option is set, it dynamically overrides `s3uploader/s3_prefix`. If the specified resource attribute exists in the data, its value will be used as the prefix; otherwise, `s3uploader/s3_prefix` will serve as the fallback. @@ -144,7 +144,7 @@ In this case, logs and traces would be stored in the following path format. metric/YYYY/MM/DD/HH/mm ``` -Optionally along with `s3_partition_format` you can provide `s3_partition_timezone` as name from IANA Time Zone +Optionally along with `s3_partition_format` you can provide `s3_partition_timezone` as name from IANA Time Zone database to change default local timezone to custom, for example `UTC` or `Europe/London`. ## Base Path Configuration diff --git a/exporter/awsxrayexporter/README.md b/exporter/awsxrayexporter/README.md index c566ad1d9142d..291948a36413f 100644 --- a/exporter/awsxrayexporter/README.md +++ b/exporter/awsxrayexporter/README.md @@ -23,8 +23,8 @@ Trace IDs and Span IDs are expected to be originally generated by either AWS API propagated by them using the `X-Amzn-Trace-Id` HTTP header. However, other generation sources are supported by replacing fully-random Trace IDs with X-Ray formatted Trace IDs where necessary: -> AWS X-Ray IDs in binary are 128 bits, the same size as W3C Trace Context IDs but the string is formatted to -> “1-{8 digit hex}-{24 digit hex}“. For example, The W3C format trace ID “4bf92f3577b34da6a3ce929d0e0e4736” is +> AWS X-Ray IDs in binary are 128 bits, the same size as W3C Trace Context IDs but the string is formatted to +> “1-{8 digit hex}-{24 digit hex}“. For example, The W3C format trace ID “4bf92f3577b34da6a3ce929d0e0e4736” is > converted to the X-Ray format trace ID “1-4bf92f35-77b34da6a3ce929d0e0e4736". The `http` object is populated when the `component` attribute value is `grpc` as well as `http`. Other diff --git a/exporter/azureblobexporter/README.md b/exporter/azureblobexporter/README.md index bffe11c9fe1c5..9574de1b00db8 100644 --- a/exporter/azureblobexporter/README.md +++ b/exporter/azureblobexporter/README.md @@ -25,7 +25,6 @@ The following settings are required: - connection_string: Connection string to the endpoint. Only needed for connection_string auth type. Once provided, it'll **override** the `url` parameter to the storage account. - federated_token_file: The path of the projected service account token file, only needed when type is workload_identity. - The following settings can be optionally configured and have default values: - container: container for metrics, logs and traces. A container organizes a set of blobs, similar to a directory in a file system. More details can refer [this](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction#containers). diff --git a/exporter/azuredataexplorerexporter/README.md b/exporter/azuredataexplorerexporter/README.md index f1629f05987a4..9a07cc8c77ea6 100644 --- a/exporter/azuredataexplorerexporter/README.md +++ b/exporter/azuredataexplorerexporter/README.md @@ -124,7 +124,7 @@ This exporter maps OpenTelemetry [trace](https://opentelemetry.io/docs/referenc ### Metrics -| ADX Table column | Description / OpenTelemetry attribute +| ADX Table column | Description / OpenTelemetry attribute | | ----------------------------- |---------------------------------------------------------------------------------------------------| | Timestamp | The timestamp of the datapoint | | MetricName | The name of the datapoint | diff --git a/exporter/azuremonitorexporter/AUTHENTICATION.md b/exporter/azuremonitorexporter/AUTHENTICATION.md index 8445df0189266..bc09dfb3bf6f1 100644 --- a/exporter/azuremonitorexporter/AUTHENTICATION.md +++ b/exporter/azuremonitorexporter/AUTHENTICATION.md @@ -75,5 +75,3 @@ azuremonitor-ingestion-proxy: - `AAD_CLIENT_ID`: client id of the service principal representing the AAD identity to use. - `AAD_TENANT_ID`: id of the AAD Tenant the service principal exists in. - `AAD_CLIENT_CERTIFICATE_PATH`: path to the .pem certificate file containing the CERTIFICATE and PRIVATE KEY parts of the certificate registered with the service principal. - - diff --git a/exporter/azuremonitorexporter/README.md b/exporter/azuremonitorexporter/README.md index 1df42f6c5d2e9..c686020795f98 100644 --- a/exporter/azuremonitorexporter/README.md +++ b/exporter/azuremonitorexporter/README.md @@ -123,7 +123,7 @@ This exporter saves log records to Application Insights `traces` table. #### Custom Events -When `custom_events_enabled` = `true`, azure monitor exporter will export log record to custom events when there's attribute `microsoft.custom_event.name` or `APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE`. +When `custom_events_enabled` = `true`, azure monitor exporter will export log record to custom events when there's attribute `microsoft.custom_event.name` or `APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE`. #### Exceptions diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 8f29247570362..e9c8e2fcf1378 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -15,7 +15,6 @@ [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib - This exporter supports sending OpenTelemetry data to [ClickHouse](https://clickhouse.com/). > ClickHouse is an open-source, high performance columnar OLAP database management system for real-time analytics using > SQL. @@ -413,7 +412,6 @@ The `CREATE TABLE` statements will always have the latest columns. In some cases the table changes will not be backwards compatible. Be sure to check the changelog for breaking changes before upgrading your collector. - ### Optional table upgrades As mentioned in the previous section, the exporter is able to detect which columns are present on the schema for backwards compatibility. diff --git a/exporter/coralogixexporter/README.md b/exporter/coralogixexporter/README.md index 8d55a827cede6..349588941de08 100644 --- a/exporter/coralogixexporter/README.md +++ b/exporter/coralogixexporter/README.md @@ -149,7 +149,7 @@ exporters: ### Application and SubSystem attributes -v0.62.0 release of OpenTelemetry Collector allows you to map Application name and Subsystem name to Resource attributes. +v0.62.0 release of OpenTelemetry Collector allows you to map Application name and Subsystem name to Resource attributes. You need to set `application_name_attributes` and `subsystem_name_attributes` fields with a list of potential Resource attributes for the AppName and Subsystem values. The first not-empty Resource attribute is going to be used. If multiple resource attributes are available, **the order of the attributes in the list determines their priority.** ### Kubernetes attributes @@ -174,7 +174,7 @@ exporters: OpenTelemetry Collector [resourcedetection](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor) processor can discover Host Resource attributes, such as `host.name` and provide Resource attributes using environment variables, which can be used for setting AppName and SubSystem fields in Coralogix. -Example: +Example: ```yaml processors: resourcedetection/system: @@ -203,7 +203,7 @@ exporters: OpenTelemetry Collector [resourcedetection](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor) processor can discover EC2 Resource attributes, such as EC2 tags as resource attributes. -Example: +Example: ```yaml processors: resourcedetection/ec2: @@ -281,7 +281,7 @@ processors: - 'attributes["your_label"] != "teamB"' ``` -This configuration ensures separate processor per each team. Any data points without an attribute for a particular team will be dropped from exporting. +This configuration ensures separate processor per each team. Any data points without an attribute for a particular team will be dropped from exporting. Secondly, set up an individual exporter per each team: ```yaml @@ -373,7 +373,6 @@ coralogix: This configuration ensures proper authentication with the Coralogix backend. Prior versions (v0.126.0 and earlier) and subsequent versions (v0.128.0 and later) are not affected by this authentication issue. - ### Need help? Our world-class customer success team is available 24/7 to walk you through the setup for this exporter and answer any questions that may come up. diff --git a/exporter/datadogexporter/README.md b/exporter/datadogexporter/README.md index 8b9ec4dbb60a7..60a0c8d0c094b 100644 --- a/exporter/datadogexporter/README.md +++ b/exporter/datadogexporter/README.md @@ -41,7 +41,6 @@ exporters: flush_timeout: 10s ``` - If you are using the batch processor instead, try lowering `send_batch_size` and `send_batch_max_size` in your config. You might want to have a separate batch processor dedicated for datadog exporter if other exporters expect a larger batch size, e.g. ``` processors: diff --git a/exporter/datasetexporter/README.md b/exporter/datasetexporter/README.md index ba51017b10223..f476edb8d7aac 100644 --- a/exporter/datasetexporter/README.md +++ b/exporter/datasetexporter/README.md @@ -27,8 +27,8 @@ If you do not want to specify `api_key` in the file, you can use the [builtin fu ### Server Host Settings -Specifying the server host is crucial for ensuring the correct functionality of DataSet. -DataSet expects the server host value to be provided in the `serverHost` attribute. +Specifying the server host is crucial for ensuring the correct functionality of DataSet. +DataSet expects the server host value to be provided in the `serverHost` attribute. If the server host value is stored in a different attribute, you can use the [resourceprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourceprocessor/README.md) or [attributesprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/attributesprocessor) to copy it into the `serverHost` attribute. You can also utilize the `server_host` settings (described below) to populate the serverHost attribute with different values. @@ -74,7 +74,6 @@ Make sure to provide the appropriate server host value in the `serverHost` attri - `sending_queue`: See [sending_queue](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) - `timeout`: See [timeout](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) - ### Attributes Enabled attributes are exported in the order: @@ -202,7 +201,6 @@ Then the event will look like: Field names can have `.` dots, `_` underscores, and `-` hyphens. You must escape slashes in Search and PowerQueries. For example, search the field name `app.kubernetes.io/component` as `app.kubernetes.io\/component`. - ### Example ```yaml @@ -275,7 +273,7 @@ Based on the given configuration and scenarios, here's the expected behavior: * Since the attribute `container_id` is set, `attributesprocessor` will copy this value to the `serverHost`. * Used `serverHost` will be `cont-pay-01`. 2. Resource: `{'node_id': 'node-pay-01', 'host.name': 'host-pay-01'}`, Log: `{'attribute.foo': 'Bar'}`, Env: `SERVER_HOST='server-pay-01'`, Hostname: `ip-172-31-27-19` - * Since the resource attribute `node_id` is set, `resourceprocessor` will copy this value to the `serverHost`. + * Since the resource attribute `node_id` is set, `resourceprocessor` will copy this value to the `serverHost`. * Used `serverHost` will be `node-pay-01`. 3. Resource: `{'host.name': 'host-pay-01'}`, Log: `{'attribute.foo': 'Bar'}`, Env: `SERVER_HOST='server-pay-01'`, Hostname: `ip-172-31-27-19` * Since the resource attribute `host.name` is set, it will be used. diff --git a/exporter/dorisexporter/README.md b/exporter/dorisexporter/README.md index bf1bfb0c84594..ec34e3fb8d6fc 100644 --- a/exporter/dorisexporter/README.md +++ b/exporter/dorisexporter/README.md @@ -12,7 +12,7 @@ [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -This exporter supports sending traces, metrics, and logs data to [Apache Doris](https://doris.apache.org/) (version >= 2.1.1). +This exporter supports sending traces, metrics, and logs data to [Apache Doris](https://doris.apache.org/) (version >= 2.1.1). ## Configuration diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 1554da55a6c73..92f85be5939cc 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -110,11 +110,11 @@ where `data_stream.type` is `logs` for log records, `metrics` for data points, a In a special case with `mapping::mode: bodymap`, `data_stream.type` field (valid values: `logs`, `metrics`) can be dynamically set from attributes. The resulting documents will contain the corresponding `data_stream.*` fields, see restrictions applied to [Data Stream Fields](https://www.elastic.co/guide/en/ecs/current/ecs-data_stream.html). 1. `data_stream.dataset` or `data_stream.namespace` in attributes (precedence: log record / data point / span attribute > scope attribute > resource attribute) - 2. Otherwise, if a scope attribute with the name `encoding.format` exists and contains a string value, `data_stream.dataset` will be set to this value. + 2. Otherwise, if a scope attribute with the name `encoding.format` exists and contains a string value, `data_stream.dataset` will be set to this value. - Note that while enabled by default, this behaviour is considered experimental. Some encoding extensions set this field (e.g. [awslogsencodingextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/encoding/awslogsencodingextension)), but it is not yet part of Semantic Conventions. There is the potential that the name of this routing field evolves as the [discussion progresses in SemConv](https://github.com/open-telemetry/semantic-conventions/issues/2854). + Note that while enabled by default, this behaviour is considered experimental. Some encoding extensions set this field (e.g. [awslogsencodingextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/encoding/awslogsencodingextension)), but it is not yet part of Semantic Conventions. There is the potential that the name of this routing field evolves as the [discussion progresses in SemConv](https://github.com/open-telemetry/semantic-conventions/issues/2854). 3. Otherwise, if scope name matches regex `/receiver/(\w*receiver)` or `/connector/(\w*connector)`, `data_stream.dataset` will be capture group #1 - 4. Otherwise, `data_stream.dataset` falls back to `generic` and `data_stream.namespace` falls back to `default`. + 4. Otherwise, `data_stream.dataset` falls back to `generic` and `data_stream.namespace` falls back to `default`. [^3]: See additional handling in [Document routing exceptions for OTel data mode](#document-routing-exceptions-for-otel-data-mode) @@ -148,8 +148,6 @@ This can be customised through the following settings: - `traces_dynamic_id` (optional): Dynamically determines the document ID to be used in Elasticsearch based on a span attribute. - `enabled`(default=false): Enable/Disable dynamic ID for spans. If `elasticsearch.document_id` exists and is not an empty string in the span attributes, it will be used as the document ID. For span events, this only applies when using `otel` mapping mode (where span events are stored as separate documents). Otherwise, the document ID will be generated by Elasticsearch. The attribute `elasticsearch.document_id` is removed from the final document when the `otel` mapping mode is used. See [Setting a document id dynamically](#setting-a-document-id-dynamically). - - #### Document routing exceptions for OTel data mode In OTel mapping mode (`mapping::mode: otel`), there is special handling in addition to the above document routing rules in [Elasticsearch document routing](#elasticsearch-document-routing). @@ -161,8 +159,6 @@ The order to determine the routing mode is the same as [Elasticsearch document r - For all documents, `data_stream.dataset` will always be appended with `.otel`. - A special case to (3)(1) in [Elasticsearch document routing](#elasticsearch-document-routing), span events are separate documents that have `data_stream.type: logs` and are routed using data stream attributes (precedence: span event attribute > scope attribute > resource attribute) - - ### Elasticsearch document mapping The Elasticsearch exporter supports several document schemas and preprocessing @@ -175,7 +171,6 @@ behaviours, which may be configured through the following settings: The mapping mode can be controlled via the client metadata key `X-Elastic-Mapping-Mode`, e.g. via HTTP headers, gRPC metadata. - It is possible to restrict which mapping modes may be requested by configuring `mapping::allowed_modes`, which defaults to all mapping modes. Keep in mind that not all processors or exporter configurations will maintain client metadata. @@ -449,7 +444,6 @@ The Elasticsearch bulk API accepts a [filter_path](https://www.elastic.co/docs/r If you want to change the `filter_path` you may do so by setting `bulk_response_filter_path` to the desired string in the configuration. - > [!NOTE] > If `items.*._index.items` is not in the BulkResponseFilterPath than for any failed documents, the exporter will not be able to log the index to which the document was being written to. @@ -488,7 +482,6 @@ exporters: [ecs]: https://www.elastic.co/guide/en/ecs/current/index.html [SemConv]: https://github.com/open-telemetry/semantic-conventions - ## ECS Mapping `elasticsearchexporter` follows ECS mapping defined here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model-appendix.md#elastic-common-schema @@ -500,10 +493,10 @@ If the target ECS field name is specified as an empty string (`""`), the convert When "Preserved" is true, the attribute will be preserved in the payload and duplicated as mapped to its ECS equivalent. When more than one SemConv attribute maps to the same ECS attribute, the converter will map all attributes to the same ECS name. -This is mean to support backwards compatibility for SemConv attributes that have been renamed/deprecated. +This is mean to support backwards compatibility for SemConv attributes that have been renamed/deprecated. The value of the last-mapped attribute will take precedence. -It is recommended to enrich events using the [elasticapmprocessor](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticapmprocessor) to ensure index documents contain all required Elastic fields to power the Kibana UI. +It is recommended to enrich events using the [elasticapmprocessor](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticapmprocessor) to ensure index documents contain all required Elastic fields to power the Kibana UI. ### Resource attribute mapping @@ -568,7 +561,6 @@ It is recommended to enrich events using the [elasticapmprocessor](https://githu | db.query.text | span.db.statement | false | | http.response.body.size | http.response.encoded_body_size | false | - ### Compound Mapping There are ECS fields that are not mapped easily 1 to 1 but require more advanced logic. diff --git a/exporter/elasticsearchexporter/documentation.md b/exporter/elasticsearchexporter/documentation.md index e68230594f8f7..e05afb1677fd9 100644 --- a/exporter/elasticsearchexporter/documentation.md +++ b/exporter/elasticsearchexporter/documentation.md @@ -84,7 +84,6 @@ In contrast, `elasticsearch.docs.retried` counts documents retried individually within otherwise successful bulk requests due to document-specific failures such as mapping conflicts. - | Unit | Metric Type | Value Type | Monotonic | Stability | | ---- | ----------- | ---------- | --------- | --------- | | 1 | Sum | Int | true | Alpha | diff --git a/exporter/faroexporter/README.md b/exporter/faroexporter/README.md index 31067968c39ed..0b08c621bd15c 100644 --- a/exporter/faroexporter/README.md +++ b/exporter/faroexporter/README.md @@ -78,4 +78,4 @@ Several helper files are leveraged to provide additional capabilities automatica - [HTTP client settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/confighttp/README.md#client-configuration) - [TLS and mTLS settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) -- [Queuing, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) +- [Queuing, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) diff --git a/exporter/fileexporter/README.md b/exporter/fileexporter/README.md index c3f45ad826d3e..ba2e9f5bcb96f 100644 --- a/exporter/fileexporter/README.md +++ b/exporter/fileexporter/README.md @@ -45,7 +45,7 @@ docker run -v "./file-exporter:/file-exporter:rwz" -v "otel-collector-config.yam ``` Note this same syntax for volumes will work with docker-compose. -You could also modify the base image and manually build your own container to have a writeable directory or change the runas uid if needed, but this is more involved. +You could also modify the base image and manually build your own container to have a writeable directory or change the runas uid if needed, but this is more involved. ## Configuration options: @@ -66,7 +66,7 @@ The following settings are optional: - `encoding`[default: none]: if specified, uses an encoding extension to encode telemetry data. Overrides `format`. - `append`[default: `false`] defines whether append to the file (`true`) or truncate (`false`). If `append: true` is set then setting `rotation` is currently not supported. - `compression`[no default]: the compression algorithm used when exporting telemetry data to file. Supported compression algorithms:`zstd` -- `flush_interval`[default: 1s]: `time.Duration` interval between flushes. See [time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for valid formats. +- `flush_interval`[default: 1s]: `time.Duration` interval between flushes. See [time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for valid formats. NOTE: a value without unit is in nanoseconds and `flush_interval` is ignored and writes are not buffered if `rotation` is set. - `create_directory`[default: false]: when set, the exporter will create the parent directory of the configured `path` if it does not exist. @@ -81,7 +81,7 @@ NOTE: a value without unit is in nanoseconds and `flush_interval` is ignored and Telemetry data is exported to a single file by default. `fileexporter` only enables file rotation when the user specifies `rotation:` in the config. However, if specified, related default settings would apply. -Telemetry is first written to a file that exactly matches the `path` setting. +Telemetry is first written to a file that exactly matches the `path` setting. When the file size exceeds `max_megabytes` or age exceeds `max_days`, the file will be rotated. When a file is rotated, **it is renamed by putting the current time in a timestamp** @@ -92,11 +92,11 @@ For example, if your `path` is `data.json` and rotation is triggered, this file ## File Compression Telemetry data is compressed according to the `compression` setting. -`fileexporter` does not compress data by default. +`fileexporter` does not compress data by default. Currently, `fileexporter` support the `zstd` compression algorithm, and we will support more compression algorithms in the future. -## File Format +## File Format Telemetry data is encoded according to the `format` setting and then written to the file. @@ -142,7 +142,7 @@ exporters: ## Get Started in an existing cluster We will follow the [documentation](https://opentelemetry.io/docs/k8s-operator/) to first install the operator in an existing cluster -and then create an OpenTelemetry Collector (otelcol) instance, +and then create an OpenTelemetry Collector (otelcol) instance, mounting an additional volume under `/data` under which the file exporter will write `metrics.json`: ``` shell kubectl apply -f - < will continue to work, but a deprecation warning will be logged at startup. Please update your > configuration to use `honeycomb_marker:`. -This exporter allows creating [markers](https://docs.honeycomb.io/working-with-your-data/markers/), via the [Honeycomb Markers API](https://docs.honeycomb.io/api/tag/Markers#operation/createMarker), based on the look of incoming telemetry. +This exporter allows creating [markers](https://docs.honeycomb.io/working-with-your-data/markers/), via the [Honeycomb Markers API](https://docs.honeycomb.io/api/tag/Markers#operation/createMarker), based on the look of incoming telemetry. The following configuration options are supported: * `api_key` (Required): This is the API key for your Honeycomb account. * `api_url` (Optional): This sets the hostname to send marker data to. If not set, will default to `https://api.honeycomb.io/` -* `markers` (Required): This is a list of configurations to create an event marker. +* `markers` (Required): This is a list of configurations to create an event marker. * `type` (Required): Specifies the marker type. - * `rules` (Required): This is a list of [OTTL](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl) rules that determine when to create an event marker. + * `rules` (Required): This is a list of [OTTL](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl) rules that determine when to create an event marker. * `log_conditions` (Required): A list of [OTTL log](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl/contexts/ottllog) conditions that determine a match. The marker will be created if **ANY** condition matches. * `dataset_slug` (Optional): The dataset in which to create the marker. If not set, will default to `__all__`. * `message_key` (Optional): The key of the attribute whose value will be used as the marker's message. If necessary the value will be converted to a string. diff --git a/exporter/kafkaexporter/README.md b/exporter/kafkaexporter/README.md index 54d440b8b34b2..dcd901884ada0 100644 --- a/exporter/kafkaexporter/README.md +++ b/exporter/kafkaexporter/README.md @@ -169,7 +169,7 @@ The destination topic can be defined in a few different ways and takes priority ## Partitioning Kafka Records -The exporter supports multiple strategies to control how records are distributed across kafka partitions within a topic. +The exporter supports multiple strategies to control how records are distributed across kafka partitions within a topic. Available strategies for partitioning are `sticky_key`, `sticky`, `round_robin`, `least_backup` and `extension` diff --git a/exporter/logicmonitorexporter/README.md b/exporter/logicmonitorexporter/README.md index b220d66638f8a..38bd125b5895a 100644 --- a/exporter/logicmonitorexporter/README.md +++ b/exporter/logicmonitorexporter/README.md @@ -69,4 +69,4 @@ But, this is not the default behaviour. In order to make the resource mapping fl resource_mapping_op: "OR" ``` -The value for `resource_mapping_op` can be `AND` or `OR`. The values are case-insensitive. \ No newline at end of file +The value for `resource_mapping_op` can be `AND` or `OR`. The values are case-insensitive. \ No newline at end of file diff --git a/exporter/logzioexporter/README.md b/exporter/logzioexporter/README.md index b0f7a58287037..e3959ee12a1b8 100644 --- a/exporter/logzioexporter/README.md +++ b/exporter/logzioexporter/README.md @@ -19,7 +19,7 @@ Logz.io exporter is utilizing opentelemetry [exporter helper](https://github.com - `account_token` (Required): Your logz.io account token for your tracing or logs account. - `region` Your logz.io account [region code](https://docs.logz.io/user-guide/accounts/account-region.html#available-regions). Defaults to `us`. Required only if your logz.io region is different than US. - `endpoint` Custom endpoint, mostly used for dev or testing. This will override the region parameter. -- `retry_on_failure` +- `retry_on_failure` - `enabled` (default = true) - `initial_interval`: Time to wait after the first failure before retrying; ignored if `enabled` is `false` (default = 5s) - `max_interval`: Is the upper bound on backoff; ignored if `enabled` is `false` (default = 30s) @@ -158,5 +158,5 @@ service: ``` #### Scope Name -When using the logs exporter with logs originating from instrumentation library (i.e opentelemetry log4j2 appender), the scopeName field will be added (if the field is populated in the original log). +When using the logs exporter with logs originating from instrumentation library (i.e opentelemetry log4j2 appender), the scopeName field will be added (if the field is populated in the original log). ``` diff --git a/exporter/mezmoexporter/README.md b/exporter/mezmoexporter/README.md index d832419d48cb4..6ed6543d08855 100644 --- a/exporter/mezmoexporter/README.md +++ b/exporter/mezmoexporter/README.md @@ -27,7 +27,7 @@ that adds `hostname` detection support. # Configuration options: -- `ingest_url` (optional): Specifies the URL to send ingested logs to. If not +- `ingest_url` (optional): Specifies the URL to send ingested logs to. If not specified, will default to `https://logs.mezmo.com/otel/ingest/rest`. - `ingest_key` (required): Ingestion key used to send log data to Mezmo. See [Ingestion Keys](https://docs.mezmo.com/docs/ingestion-key) for more details. diff --git a/exporter/opensearchexporter/README.md b/exporter/opensearchexporter/README.md index d5eeda1ffb137..779bfeb430b79 100644 --- a/exporter/opensearchexporter/README.md +++ b/exporter/opensearchexporter/README.md @@ -98,7 +98,6 @@ If any placeholder key is missing, the fallback value is used e.g.: ### OpenSearch document mapping - The mapping mode can be controlled via the scope attribute `opensearch.mapping.mode`. The OpenSearch exporter supports several document schemas and preprocessing behaviors, which may be configured through the following settings: diff --git a/exporter/prometheusremotewriteexporter/DESIGN.md b/exporter/prometheusremotewriteexporter/DESIGN.md index 32ece531ec79f..00a948b6285d8 100644 --- a/exporter/prometheusremotewriteexporter/DESIGN.md +++ b/exporter/prometheusremotewriteexporter/DESIGN.md @@ -1,5 +1,4 @@ - # **OpenTelemetry Collector Prometheus Remote Write/Cortex Exporter Design** Authors: @huyan0, @danielbang907 @@ -24,7 +23,6 @@ The Prometheus remote write/Cortex exporter should write metrics to a remote URL TimeSeries stores its metric name in its labels and does not describe metric types or start timestamps. To convert to TimeSeries data, buckets of a Histogram are broken down into individual TimeSeries with a bound label(`le`), and a similar process happens with quantiles in a Summary. - More details of Prometheus remote write API can be found in Prometheus [documentation](https://prometheus.io/docs/prometheus/latest/storage/#overview) and Cortex [documentation](https://cortexmetrics.io/docs/api/). ### **1.2 Gaps and Assumptions** @@ -35,7 +33,7 @@ Currently, metrics from the OpenTelemetry SDKs cannot be exported to Prometheus To overcome this gap in the Collector pipeline, we had proposed 2 different solutions: 1. Add a [metric aggregation processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/4968) to the collector pipeline to aggregate delta values into cumulative values for cumulative backends. This solution requires users to set up a collector agent next to each SDK to make sure delta values are aggregated correctly. -2. Require the OTLP exporters in SDKs to [send cumulative values for cumulative metric types to the Collector by default](https://github.com/open-telemetry/opentelemetry-specification/issues/731). Therefore, no aggregation of delta metric values is required in the Collector pipeline for Prometheus/storage backends to properly process the data. +2. Require the OTLP exporters in SDKs to [send cumulative values for cumulative metric types to the Collector by default](https://github.com/open-telemetry/opentelemetry-specification/issues/731). Therefore, no aggregation of delta metric values is required in the Collector pipeline for Prometheus/storage backends to properly process the data. **Gap 2:** Another gap is that OTLP metric definition is still in development. This exporter will require refactoring as OTLP changes in the future. @@ -56,14 +54,11 @@ To group data points by label set, the exporter should create a map with each Pu * the metric name * the set of labels that identify a unique TimeSeries - The exporter should create a signature string as map key by concatenating metric type, metric name, and label names and label values at each data point. To ensure correctness, the label set at each data point should be sorted by label key before generating the signature string. An alternative key type is in the exiting label.Set implementation from the OpenTelemetry Go API. It provides a Distinct type that guarantees the result will equal the equivalent Distinct value of any label set with the same elements as this, where sets are made unique by choosing the last value in the input for any given key. If we allocate a Go API's kv.KeyValue for every label of a data point, then a label.Set from the API can be created, and its Distinct value can be used as map key. - -The value of the map should be Prometheus TimeSeries, and each data point’s value and timestamp should be inserted to its corresponding TimeSeries in the map as a Sample, each metric’s label set and metric name should be combined and translated to a Prometheus label set; a new TimeSeries should be created if the string signature is not in the map. - +The value of the map should be Prometheus TimeSeries, and each data point’s value and timestamp should be inserted to its corresponding TimeSeries in the map as a Sample, each metric’s label set and metric name should be combined and translated to a Prometheus label set; a new TimeSeries should be created if the string signature is not in the map. Pseudocode: @@ -73,14 +68,14 @@ Pseudocode: map := make(map[String][]TimeSeries) for metric in metricsData: - for point in metric: - // Generate signature string - sig := pointSignature(metric, point) + for point in metric: + // Generate signature string + sig := pointSignature(metric, point) - // Find corresponding TimeSeries in map - // Add to TimeSeries + // Find corresponding TimeSeries in map + // Add to TimeSeries - // Sends TimeSeries to backend + // Sends TimeSeries to backend export(map) } @@ -88,12 +83,10 @@ Pseudocode: Each Prometheus remote write TimeSeries represents less semantic information than an OTLP metric. The temporality property of a OTLP metric is ignored in a TimeSeries because it is always considered as cumulative for monotonic types and histogram, and the type property of a OTLP metric is translated by mapping each metric to one or multiple TimeSeries. The following sections explain how to map each OTLP metric type to Prometheus remote write TimeSeries. - **INT64, MONOTONIC_INT64, DOUBLE, MONOTONIC_DOUBLE** Each unique label set within metrics of these types can be converted to exactly one TimeSeries. From the perspective of Prometheus client types, INT64 and DOUBLE correspond to gauge metrics, and MONOTONIC types correspond to counter metrics. In both cases, data points will be exported directly without aggregation. Any metric of the monotonic types that is not cumulative should be dropped; non-monotonic scalar types are assumed to represent gauge values, thus its temporality is not checked. Monotonic types need to have a `_total` suffix in its metric name when exporting; this is a requirement of Prometheus. - **HISTOGRAM** Each histogram data point can be converted to 2 + n + 1 Prometheus remote write TimeSeries: @@ -105,7 +98,6 @@ Each histogram data point can be converted to 2 + n + 1 Prometheus remote write Prometheus bucket values are cumulative, meaning the count of each bucket should contain counts from buckets with lower bounds. In addition, Exemplars from a histogram data point are ignored. When adding a bucket of the histogram data point to the map, the string signature should also contain a `le` label that indicates the bound value. This label should also be exported. Any histogram metric that is not cumulative should be dropped. - **SUMMARY** Each summary data point can be converted to 2 + n Prometheus remote write TimeSeries: @@ -120,24 +112,21 @@ When adding a quantile of the summary data point to the map, the string signatur The Prometheus remote write/Cortex exporter should call proto.Marshal() to convert multiple TimeSeries to a byte array. Then, the exporter should send the byte array to Prometheus remote storage in a HTTP request. - Authentication credentials should be added to each request before sending to the backend. Basic auth and bearer token headers can be added using Golang http.Client’s default configuration options. Other authentication headers can be added by implementing a client interceptor. - Pseudocode: - func export(*map) error { - // Stores timeseries - arr := make([]TimeSeries) + // Stores timeseries + arr := make([]TimeSeries) - for timeseries in map: - arr = append(arr, timeseries) + for timeseries in map: + arr = append(arr, timeseries) - // Converts arr to WriteRequest - request := proto.Marshal(arr) + // Converts arr to WriteRequest + request := proto.Marshal(arr) - // Sends HTTP request to endpoint + // Sends HTTP request to endpoint } ## **3. Other Components** @@ -146,10 +135,8 @@ Pseudocode: This struct is based on an inputted YAML file at the beginning of the pipeline and defines the configurations for an Exporter build. Examples of configuration parameters are HTTP endpoint, compression type, backend program, etc. - Converting YAML to a Go struct is done by the Collector, using [_the Viper package_](https://github.com/spf13/viper), which is an open-source library that seamlessly converts inputted YAML files into a usable, appropriate Config struct. - An example of the exporter section of the Collector config.yml YAML file can be seen below: ... @@ -209,10 +196,8 @@ An example of the exporter section of the Collector config.yml YAML file can be This struct implements the ExporterFactory interface, and is used during collector’s pipeline initialization to create the Exporter instances as defined by the Config struct. The `exporterhelper` package will be used to create the exporter and the factory. - Our Factory type will look very similar to other exporters’ factory implementation. For our implementation, our Factory instance will implement three methods - **Methods** NewFactory @@ -222,18 +207,15 @@ This method will use the NewFactory method within the `exporterhelper` package t This method creates the default configuration for Prometheus remote write/Cortex exporter. - createMetricsExporter -This method constructs a new http.Client with interceptors that add headers to any request it sends. Then, this method initializes a new Prometheus remote write exporter/Cortex exporter with the http.Client. This method constructs a collector Prometheus remote write/Cortex exporter with the created SDK exporter - - +This method constructs a new http.Client with interceptors that add headers to any request it sends. Then, this method initializes a new Prometheus remote write exporter/Cortex exporter with the http.Client. This method constructs a collector Prometheus remote write/Cortex exporter with the created SDK exporter ## **4. Other Considerations** ### **4.1 Concurrency** -The Prometheus remote write/Cortex exporter should be thread-safe; In this design, the only resource shared across goroutines is the http.Client from the Golang library. It is thread-safe, thus, our code is thread-safe. +The Prometheus remote write/Cortex exporter should be thread-safe; In this design, the only resource shared across goroutines is the http.Client from the Golang library. It is thread-safe, thus, our code is thread-safe. ### **4.2 Shutdown Behavior** @@ -245,14 +227,14 @@ Once the shutdown() function is called, the exporter should stop accepting incom } func PushMetrics() { - select: - case <- stopCh - return error - default: - waitGroup.Add(1) - defer waitGroup.Done() - // export metrics - ... + select: + case <- stopCh + return error + default: + waitGroup.Add(1) + defer waitGroup.Done() + // export metrics + ... } ### **4.3 Timeout Behavior** @@ -269,20 +251,13 @@ Users should be able to pass in a time for the each http request as part of the ### **4.4 Error Behavior** -The PushMetricsData() function should return the number of dropped metrics. Any monotonic and histogram metrics that are not cumulative should be dropped. This can be done by checking the temporality of each received metric. Any error should be returned to the caller, and the error message should be descriptive. - - +The PushMetricsData() function should return the number of dropped metrics. Any monotonic and histogram metrics that are not cumulative should be dropped. This can be done by checking the temporality of each received metric. Any error should be returned to the caller, and the error message should be descriptive. ### **4.5 Test Strategy** We will follow test-driven development practices while completing this project. We’ll write unit tests before implementing production code. Tests will cover normal and abnormal inputs and test for edge cases. We will provide end-to-end tests using mock backend/client. Our target is to get 90% or more of code coverage. - - ## **Request for Feedback** We'd like to get some feedback on whether we made the appropriate assumptions in [this](#12-gaps-and-assumptions) section, and appreciate more comments, updates , and suggestions on the topic. Please let us know if there are any revisions, technical or informational, necessary for this document. Thank you! - - - diff --git a/exporter/prometheusremotewriteexporter/README.md b/exporter/prometheusremotewriteexporter/README.md index 4170486d73dd8..575e4cbc56635 100644 --- a/exporter/prometheusremotewriteexporter/README.md +++ b/exporter/prometheusremotewriteexporter/README.md @@ -62,22 +62,21 @@ The following settings can be optionally configured: - `wal`: Write-Ahead-Log settings for the exporter. - `directory` (default = ``): The directory to store the WAL in. - `buffer_size` (default = `300`): Count of elements to be read from the WAL before truncating. - - `truncate_frequency` (default = `1m`): Frequency for how often the WAL should be truncated. - - `lag_record_frequency` (default = `15s`): Frequency for how often the exporter will record the lag of the WAL. + - `truncate_frequency` (default = `1m`): Frequency for how often the WAL should be truncated. + - `lag_record_frequency` (default = `15s`): Frequency for how often the exporter will record the lag of the WAL. - `target_info`: customize `target_info` metric - `enabled` (default = true): If `enabled` is `true`, a `target_info` metric will be generated for each resource metric (see https://github.com/open-telemetry/opentelemetry-specification/pull/2381). - `disable_scope_info` (default = false): If `true`, the scope info labels (`otel_scope_name`, `otel_scope_version` and `otel_scope_...` attributes) will not be exported. -- `max_batch_size_bytes` (default = `3000000` -> `~2.861 mb`): Maximum size of a batch of samples to be sent to the remote +- `max_batch_size_bytes` (default = `3000000` -> `~2.861 mb`): Maximum size of a batch of samples to be sent to the remote write endpoint. If the batch size is larger than this value, it will be split into multiple batches. This option is ignored when using the wal and where the wal buffer_size / truncate_frequency will be used. -- `max_batch_request_parallelism` (default = `5`): Maximum parallelism allowed when sending multiple requests to the remote write endpoint. - If the remote write endpoint does not support out of order samples, this should be set to `1`. -- `protobuf_message` (default = `prometheus.WriteRequest`): +- `max_batch_request_parallelism` (default = `5`): Maximum parallelism allowed when sending multiple requests to the remote write endpoint. + If the remote write endpoint does not support out of order samples, this should be set to `1`. +- `protobuf_message` (default = `prometheus.WriteRequest`): - Protobuf message to use when writing to the remote write endpoint. This option is ignored unless the `exporter.prometheusremotewritexporter.enableSendingRW2` feature gate is enabled. - `prometheus.WriteRequest` is the message used in [Remote Write 1.0](https://prometheus.io/docs/specs/remote_write_spec/). - `io.prometheus.write.v2.Request` is the message used in [Remote Write 2.0](https://prometheus.io/docs/specs/remote_write_spec_2_0/). It is more efficient, always includes metadata, and adds support for the created timestamp and native histograms. Your remote storage provider must support PRW 2.0 to be able to use this message. PRW 2.0 support is currently **In Development** and is only partially implemented, thus, not ready for usage. - Example: ```yaml diff --git a/exporter/pulsarexporter/README.md b/exporter/pulsarexporter/README.md index 45ff2f9a5ef35..addfcfa044f0e 100644 --- a/exporter/pulsarexporter/README.md +++ b/exporter/pulsarexporter/README.md @@ -50,7 +50,7 @@ The following settings can be optionally configured: - `zts_url`: - `producer` - `max_reconnect_broker`: specifies the maximum retry number of reconnectToBroker. (default: ultimate) - - `hashing_scheme`: used to define the partition on where to publish a particular message. Can be set to `java_string_hash` (default) or `murmur3_32hash`. + - `hashing_scheme`: used to define the partition on where to publish a particular message. Can be set to `java_string_hash` (default) or `murmur3_32hash`. - `compression_level`: one of 'default' (default), 'faster', or 'better'. - `compression_type`: one of 'none' (default), 'lz4', 'zlib', or 'zstd'. - `max_pending_messages`: specifies the max size of the queue holding the messages pending to receive an acknowledgment from the broker. diff --git a/exporter/rabbitmqexporter/README.md b/exporter/rabbitmqexporter/README.md index d611a72286cce..a9526a8b8773b 100644 --- a/exporter/rabbitmqexporter/README.md +++ b/exporter/rabbitmqexporter/README.md @@ -15,7 +15,7 @@ Exports metrics, traces, and logs to [RabbitMQ](https://www.rabbitmq.com/) using the AMQP 0.9.1 protocol. -Messages are published to the [default exchange](https://www.rabbitmq.com/tutorials/amqp-concepts#exchange-default) direct exchange, but optionally can be published to a different direct exchange. +Messages are published to the [default exchange](https://www.rabbitmq.com/tutorials/amqp-concepts#exchange-default) direct exchange, but optionally can be published to a different direct exchange. This component expects that exchanges, queues, and bindings already exist - they are not currently created by this component. diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md index 73de16028c365..dac4afc42cf30 100644 --- a/exporter/sematextexporter/README.md +++ b/exporter/sematextexporter/README.md @@ -28,7 +28,7 @@ The following configuration options are supported: * `max_elapsed_time` (default = 120s) Maximum amount of time (including retries) spent trying to send a request/batch * `region` **(required)** Region specifies the Sematext region the user is operating in; must be one of: * `US` - * `EU` + * `EU` * `metrics.app_token` **(required if sending metrics)** Token of the Sematext Monitoring App to which metrics data will be sent. Must be a valid UUID string in the format `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`. For example: `2046e37c-4fac-45f6-831d-922d43fde759`. * `metrics.payload_max_lines` (default = 1_000) Maximum number of lines allowed per HTTP POST request * `metrics.payload_max_bytes` (default = 300_000) Maximum number of bytes allowed per HTTP POST request diff --git a/exporter/signalfxexporter/README.md b/exporter/signalfxexporter/README.md index 1c1a6423477a7..ae893a53b17b2 100644 --- a/exporter/signalfxexporter/README.md +++ b/exporter/signalfxexporter/README.md @@ -40,8 +40,8 @@ The following configuration options are required: `realm` is set, this option is derived and will be `https://ingest.{realm}.observability.splunkcloud.com`. If a value is explicitly set, the value of `realm` will not be used in determining - `ingest_url`. The explicit value will be used instead. The exporter will - automatically append the appropriate path: "/v2/datapoint" for metrics, + `ingest_url`. The explicit value will be used instead. The exporter will + automatically append the appropriate path: "/v2/datapoint" for metrics, and "/v2/event" for events. The following configuration options can also be configured: @@ -55,7 +55,7 @@ The following configuration options can also be configured: origin for only this exporter, as others will reveal the organization access token by not filtering the attribute. - `exclude_metrics`: List of metric filters that will determine metrics to be - excluded from sending to Signalfx backend. The filtering is applied after the default + excluded from sending to Signalfx backend. The filtering is applied after the default translations controlled by `disable_default_translation_rules` option. See in [testdata/config.yaml](./testdata/config.yaml) for examples. Apart from the values explicitly provided via this option, by default, [default metrics](./internal/translation/default_metrics.go) are @@ -75,7 +75,7 @@ The following configuration options can also be configured: dimensions: state: [interrupt, user, system] ``` -- `log_data_points` (default = `false`): If the log level is set to `debug` +- `log_data_points` (default = `false`): If the log level is set to `debug` and this is true, all datapoints dispatched to Splunk Observability Cloud will be logged - `log_dimension_updates` (default = `false`): Whether or not to log dimension updates. @@ -86,13 +86,13 @@ The following configuration options can also be configured: complete. - `http2_read_idle_timeout` (default = 10s): Send a ping frame for a health check if the connection has been idle for the configured value. 0s means http/2 health check will be disabled. -- `http2_ping_timeout` (default = 10s): Triggered by `http2_read_idle_timeout`; When there's no response to the ping within the configured value, - the connection will be closed. If this value is set to 0, it will default to 15s. +- `http2_ping_timeout` (default = 10s): Triggered by `http2_read_idle_timeout`; When there's no response to the ping within the configured value, + the connection will be closed. If this value is set to 0, it will default to 15s. - `headers` (no default): Headers to pass in the payload. - `max_idle_conns` (default = 100): The maximum idle HTTP connections the client can keep open. - `max_idle_conns_per_host` (default = 100): The maximum idle HTTP connections the client can keep open per host. - `idle_conn_timeout` (default = 30s): The maximum amount of time an idle connection will remain open before closing itself. -- More HTTP settings are available, see +- More HTTP settings are available, see [HTTP settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/confighttp/README.md). - `sync_host_metadata`: Defines whether the exporter should scrape host metadata and send it as property updates to SignalFx backend. Disabled by default. @@ -128,14 +128,14 @@ The following configuration options can also be configured: - `idle_conn_timeout` (default = 30s): The maximum amount of time an idle connection will remain open before closing itself. - `timeout` (default = 10s): Amount of time to wait for the dimension HTTP request to complete. - `drop_tags` (default = false): Flag that indicates whether to drop the tags from the metadata sent to Splunk Observability Cloud by the exporter. -- `nonalphanumeric_dimension_chars`: (default = `"_-."`) A string of characters -that are allowed to be used as a dimension key in addition to alphanumeric -characters. Each nonalphanumeric dimension key character that isn't in this string +- `nonalphanumeric_dimension_chars`: (default = `"_-."`) A string of characters +that are allowed to be used as a dimension key in addition to alphanumeric +characters. Each nonalphanumeric dimension key character that isn't in this string will be replaced with a `_`. - `ingest_tls`: (no default) exposes a list of TLS settings to establish a secure connection with signafx receiver configured on another collector instance. - `ca_file` needs to be set if the exporter's `ingest_url` is pointing to a signalfx receiver with TLS enabled and using a self-signed certificate where its CA is not loaded in the system cert pool. - Full list of TLS options can be found in the configtls [README](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/configtls#client-configuration) + Full list of TLS options can be found in the configtls [README](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/configtls#client-configuration) The following example instructs the signalfx exporter ingest client to use a custom `ca_file` to verify the server certificate. ```yaml ingest_tls: @@ -245,7 +245,7 @@ The default rules will create the following aggregated metrics from the [`hostme * vmpage_io.swap.out In addition to the aggregated metrics, the default translation rules make available the following "per core" custom hostmetrics. -The CPU number is assigned to the dimension `cpu` +The CPU number is assigned to the dimension `cpu` * cpu.interrupt * cpu.nice diff --git a/exporter/signalfxexporter/docs/translation_rules_migration_guide.md b/exporter/signalfxexporter/docs/translation_rules_migration_guide.md index 7e498de6ed8cb..900ca08d7f514 100644 --- a/exporter/signalfxexporter/docs/translation_rules_migration_guide.md +++ b/exporter/signalfxexporter/docs/translation_rules_migration_guide.md @@ -19,7 +19,7 @@ can be used to accomplish the same functionality. | calculate_new_metric | `metricsgeneration` processor's `calculate` functionality | [calculate example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricsgenerationprocessor#example-configurations) | | convert_values | `transform` processor's `Double` or `Int` converter on a `datapoint` context | [`Double` example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl/ottlfuncs#double), [`Int` example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl/ottlfuncs#int) | | copy_metrics | `metricstransform` processor's `insert` functionality | [copy all datapoints](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricstransformprocessor#create-a-new-metric-from-an-existing-metric), [conditionally copy datapoints](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricstransformprocessor#create-a-new-metric-from-an-existing-metric-with-matching-label-values) | -| delta_metric | `cumulativetodelta` processor. To preserve original metrics, first copy the original metric, then use the copied metric in the `cumulativetodelta` processor | [specify which metrics to convert example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/cumulativetodeltaprocessor#examples) +| delta_metric | `cumulativetodelta` processor. To preserve original metrics, first copy the original metric, then use the copied metric in the `cumulativetodelta` processor | [specify which metrics to convert example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/cumulativetodeltaprocessor#examples) | | drop_dimensions | `transform` processor's `delete_keys` function with the `datapoint` context | [simple example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl/ottlfuncs#delete_key), use a `where` clause with the given example to filter based upon the metric name or dimension value | | drop_metrics | `filter` processor | [drop by name and value example](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor#dropping-specific-metric-and-value) | | multiply_int, divide_int, multiply_float | `metricstransform` processor's `scale` value functionality | [one metric](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricstransformprocessor#scale-value) | diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index 82bd15215188a..d802fd5b3a396 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -46,10 +46,10 @@ The following configuration options can also be configured: (~ 800 MB). When set to 0, it will treat as infinite length and it will create only one request per batch. - `max_event_size` (default: 5242880): Maximum raw uncompressed individual event size in bytes. Maximum allowed value is 838860800 (~ 800 MB). - `splunk_app_name` (default: "OpenTelemetry Collector Contrib") App name is used to track telemetry information for Splunk App's using HEC by App name. -- `splunk_app_version` (default: Current OpenTelemetry Collector Contrib Build Version): App version is used to track telemetry information for Splunk App's using HEC by App version. -- `log_data_enabled` (default: true): Specifies whether the log data is exported. Set it to `false` if you want the log +- `splunk_app_version` (default: Current OpenTelemetry Collector Contrib Build Version): App version is used to track telemetry information for Splunk App's using HEC by App version. +- `log_data_enabled` (default: true): Specifies whether the log data is exported. Set it to `false` if you want the log data to be dropped instead. Applicable in the `logs` pipeline only. -- `profiling_data_enabled` (default: true): Specifies whether the profiling data is exported. Set it to `false` if +- `profiling_data_enabled` (default: true): Specifies whether the profiling data is exported. Set it to `false` if you want the profiling data to be dropped instead. Applicable in the `logs` pipeline only. - `health_path` (default = '/services/collector/health'): The path reporting [health checks](https://docs.splunk.com/Documentation/Splunk/9.0.1/RESTREF/RESTinput#services.2Fcollector.2Fhealth). - `health_check_enabled` (default = false): Whether to perform Splunk HEC Health Check during the exporter's startup. @@ -148,4 +148,3 @@ Several helper files are leveraged to provide additional capabilities automatica - [HTTP settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/confighttp/README.md) - [TLS and mTLS settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) - [Queuing, retry and timeout settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) - diff --git a/exporter/stefexporter/README.md b/exporter/stefexporter/README.md index b8c4d1ee0414f..690e9f811bc98 100644 --- a/exporter/stefexporter/README.md +++ b/exporter/stefexporter/README.md @@ -17,7 +17,7 @@ Exports data via gRPC using [Otel/STEF format](https://github.com/splunk/stef/tree/main/go/otel). Otel/STEF is a compact and fast telemetry format. It is currently the fastest -benchmarked metric format in the Collector. Here are recent +benchmarked metric format in the Collector. Here are recent [benchmarking results](https://www.stefdata.net/benchmarks.html), comparing CPU usage when using a few different formats (lower is better): @@ -33,35 +33,35 @@ comparing CPU usage when using a few different formats (lower is better): STEF in this benchmark outperforms all other formats, including OTLP, which was previously known as the fastest format in the Collector. -STEF is also very compact on the wire. In uncompressed mode, it typically yields payloads +STEF is also very compact on the wire. In uncompressed mode, it typically yields payloads that are more than 10x smaller than OTLP payloads. In compressed mode, STEF is typically 5-7 times more compact than OTLP. -Here are some +Here are some [STEF benchmark results](https://www.stefdata.net/benchmarks.html), comparing STEF and OTLP payload sizes for a few sample payloads produced by Collector: - +Host and Collector Metrics Size Comparison And here is a dataset from [Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo): - +Astronomy Shop Demo Metrics Size Comparison -(This exporter implementation uses unsorted STEF format, labeled "STEF Unsorted" in +(This exporter implementation uses unsorted STEF format, labeled "STEF Unsorted" in charts above). There are currently no known formats that match STEF's compactness and performance. -STEF exporter can be used to send metrics from the Collector to any STEF-compatible +STEF exporter can be used to send metrics from the Collector to any STEF-compatible backend. STEF can also be used to send metric data between Collector instances, in which case on the receiving side a [STEF receiver](../../receiver/stefreceiver/README.md) should be used. -STEF format, STEF exporter and receiver implementations are currently in alpha -stage of development, during which **the format may undergo breaking changes.** -To ensure interoperability between sending and receiving Collectors make sure -you are using versions of exporter and receiver that are compiled with the same +STEF format, STEF exporter and receiver implementations are currently in alpha +stage of development, during which **the format may undergo breaking changes.** +To ensure interoperability between sending and receiving Collectors make sure +you are using versions of exporter and receiver that are compiled with the same version of STEF library. Feedback about STEF format and implementation is welcome in the form of issues @@ -73,8 +73,8 @@ The following settings are required: - `endpoint` (no default): host:port to which the exporter is going to send STEF metric data, using the STEF/gRPC protocol. The valid syntax is described - [here](https://github.com/grpc/grpc/blob/master/doc/naming.md). When sending to - another Collector you will typically use `:4320` as the endpoint, since + [here](https://github.com/grpc/grpc/blob/master/doc/naming.md). When sending to + another Collector you will typically use `:4320` as the endpoint, since port 4320 is the default port used by STEF receiver. If a scheme of `https` is used then client transport security is enabled and overrides the `insecure` setting. - `tls`: see [TLS Configuration Settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) diff --git a/exporter/tinybirdexporter/README.md b/exporter/tinybirdexporter/README.md index a0218ae1fcbd0..30ece809c0b2c 100644 --- a/exporter/tinybirdexporter/README.md +++ b/exporter/tinybirdexporter/README.md @@ -18,7 +18,6 @@ This exporter sends logs, metrics, and traces to [Tinybird](https://www.tinybird Tinybird is a real-time analytics platform for ingesting, transforming, and serving data with low latency and high throughput. Telemetry data sent to Tinybird can be queried and analyzed in real time using SQL.`` - ## Quick Start Looking for a quick start? Check out our [Open Telemetry Template](https://github.com/tinybirdco/tinybird-otel-template), which provides a ready-to-use example for configuring the Tinybird OpenTelemetry Exporter. @@ -43,7 +42,6 @@ Looking for a quick start? Check out our [Open Telemetry Template](https://githu - `retry_on_failure`: Configuration for retry behavior on failures - `sending_queue`: Configuration for the sending queue - ### Basic Configuration ```yaml @@ -93,7 +91,6 @@ Before using this exporter, you need to create the corresponding data sources in - Only contain letters, numbers, and underscores - Match the data source names specified in your configuration - ## Authentication The exporter uses API token authentication. You can provide the token in several ways: diff --git a/extension/ackextension/README.md b/extension/ackextension/README.md index ff151a1d76278..3b449b5fcf48a 100644 --- a/extension/ackextension/README.md +++ b/extension/ackextension/README.md @@ -4,7 +4,6 @@ This extension allows acking of data upon successful processing. The upstream agent can choose to send event again if ack fails. - | Status | | | ------------- |-----------| | Stability | [alpha] | diff --git a/extension/asapauthextension/README.md b/extension/asapauthextension/README.md index b0b8ef47ad6fb..8aba9ccfc17b4 100644 --- a/extension/asapauthextension/README.md +++ b/extension/asapauthextension/README.md @@ -1,10 +1,9 @@ # ASAP Client Authentication Extension -This extension provides [Atlassian Service Authentication Protocol](https://s2sauth.bitbucket.io/) (ASAP) client +This extension provides [Atlassian Service Authentication Protocol](https://s2sauth.bitbucket.io/) (ASAP) client credentials for HTTP or gRPC based exporters. - | Status | | | ------------- |-----------| | Stability | [beta] | @@ -48,4 +47,3 @@ exporters: auth: authenticator: asapclient ``` - diff --git a/extension/awsproxy/README.md b/extension/awsproxy/README.md index cbd47e7495cac..8517f594912ce 100644 --- a/extension/awsproxy/README.md +++ b/extension/awsproxy/README.md @@ -6,7 +6,6 @@ AWS API, applying authentication and signing. This allows applications to avoid a service, instead configuring the AWS exporter and/or proxy in the OpenTelemetry collector and only providing the collector with credentials. - | Status | | | ------------- |-----------| | Stability | [beta] | @@ -45,7 +44,6 @@ Default: `localhost:2000` See our [security best practices doc](https://opentelemetry.io/docs/security/config-best-practices/#protect-against-denial-of-service-attacks) to understand how to set the endpoint in different environments. - ### proxy_address (Optional) Defines the proxy address that this extension forwards HTTP requests to the AWS backend through. If left unconfigured, requests will be sent directly. This will generally be set to a NAT gateway when the collector is running on a network without public internet. diff --git a/extension/basicauthextension/README.md b/extension/basicauthextension/README.md index b65ac3d92f5ad..d0372a1d6bbbd 100644 --- a/extension/basicauthextension/README.md +++ b/extension/basicauthextension/README.md @@ -21,7 +21,7 @@ When used as ServerAuthenticator, if the authentication is successful `client.In - `username`: The username of the authenticated user. - `raw`: Raw base64 encoded credentials. -The configuration should specify only one instance of `basicauth` extension for either client or server authentication. +The configuration should specify only one instance of `basicauth` extension for either client or server authentication. The following are the configuration options: diff --git a/extension/bearertokenauthextension/README.md b/extension/bearertokenauthextension/README.md index 17372360651b2..c8d8d599fec70 100644 --- a/extension/bearertokenauthextension/README.md +++ b/extension/bearertokenauthextension/README.md @@ -34,7 +34,6 @@ Either one of `token` or `filename` field is required. If both are specified, th **Note**: bearertokenauth requires transport layer security enabled on the exporter. - ```yaml extensions: bearertokenauth: diff --git a/extension/encoding/README.md b/extension/encoding/README.md index 45649babbdef7..0f7aef94b9710 100644 --- a/extension/encoding/README.md +++ b/extension/encoding/README.md @@ -2,18 +2,18 @@ The encoding extensions can be used by compatible receivers or exporters to encode or decode data into/from a specific format. This is useful when the data is being sent to/from a system that expects a specific format and doesn't support -the OpenTelemetry protocol. +the OpenTelemetry protocol. _🚧 Under active development 🚧_ ## Component Milestones -To help track what work needs to be done with this component, these are the currently active goals being +To help track what work needs to be done with this component, these are the currently active goals being worked towards. ### Development -- Add encoding extensions support additionally to the existing ways of configuring encodings (where applicable) +- Add encoding extensions support additionally to the existing ways of configuring encodings (where applicable) to the following components: - `file receiver` - `file exporter` diff --git a/extension/encoding/avrologencodingextension/README.md b/extension/encoding/avrologencodingextension/README.md index ccd1c6ad90545..479f6f6c94a21 100644 --- a/extension/encoding/avrologencodingextension/README.md +++ b/extension/encoding/avrologencodingextension/README.md @@ -3,7 +3,6 @@ The `avrolog` encoding extension is used to unmarshal AVRO and insert it into the body of a log record. Marshalling is not supported. - | Status | | | ------------- |-----------| | Stability | [development] | diff --git a/extension/encoding/awscloudwatchmetricstreamsencodingextension/README.md b/extension/encoding/awscloudwatchmetricstreamsencodingextension/README.md index d5e6e825754ef..59a6143ecaaa0 100644 --- a/extension/encoding/awscloudwatchmetricstreamsencodingextension/README.md +++ b/extension/encoding/awscloudwatchmetricstreamsencodingextension/README.md @@ -3,7 +3,6 @@ This extension unmarshalls metrics encoded in formats produced by [Amazon CloudWatch Metric Streams](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Metric-Streams.html). - | Status | | | ------------- |-----------| | Stability | [alpha] | diff --git a/extension/encoding/awslogsencodingextension/README.md b/extension/encoding/awslogsencodingextension/README.md index b1e50c93b398f..dca8076d89347 100644 --- a/extension/encoding/awslogsencodingextension/README.md +++ b/extension/encoding/awslogsencodingextension/README.md @@ -21,14 +21,13 @@ This extension unmarshals logs encoded in formats produced by AWS services, incl - Parquet support still to be added. - [S3 access log records](https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html). - [AWS CloudTrail logs](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-log-file-examples.html). -- ELB access logs: + - ELB access logs: - [Classic Load Balancer (CLB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html) - [Application Load Balancer (ALB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html) - [Network Load Balancer (NLB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html) - [AWS Network Firewall logs](https://docs.aws.amazon.com/network-firewall/latest/developerguide/firewall-logging-contents.html). - (More to be added later.) - Example for Amazon CloudWatch Logs Subscription Filters: ```yaml extensions: @@ -388,7 +387,6 @@ The table below summarizes streaming support details for each log type, along wi [S3 access log record fields](https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html) are mapped this way in the resulting OpenTelemetry log: - | AWS field | OpenTelemetry Field | |---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Bucket owner | `aws.s3.owner` | @@ -421,8 +419,6 @@ The table below summarizes streaming support details for each log type, along wi [AWS WAF log record fields](https://docs.aws.amazon.com/waf/latest/developerguide/logging-fields.html) are mapped this way in the resulting OpenTelemetry log: - - | Original log field | OpenTelemetry field | |-------------------------------|--------------------------------------------------------------------------------------------------| | `webaclId` | `cloud.resource_id`
Also splits the value to get:
1.`cloud.region`
2.`cloud.account.id` | @@ -550,7 +546,6 @@ ELB access log record fields are mapped this way in the resulting OpenTelemetry > AWS Fields are according to [documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html). - | **AWS Field** | **OpenTelemetry Field(s)** | |---------------------------|---------------------------------------------------------------| | type | `network.protocol.name` | @@ -587,7 +582,6 @@ ELB access log record fields are mapped this way in the resulting OpenTelemetry | transformed_uri | `aws.elb.transformed_uri` | | request_transform_status | `aws.elb.request_transform_status` | - #### Network Load Balancer (NLB) > AWS Fields are according to [documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest//network/load-balancer-access-logs.html#access-log-entry-format). @@ -698,7 +692,7 @@ The following fields are common across all log types: #### TLS log fields -[See TLS inspection page](https://docs.aws.amazon.com/network-firewall/latest/developerguide/tls-inspection-logging.html) and [Suricata fields](https://docs.suricata.io/en/latest/output/eve/eve-json-format.html#event-type-tls) for more details. +[See TLS inspection page](https://docs.aws.amazon.com/network-firewall/latest/developerguide/tls-inspection-logging.html) and [Suricata fields](https://docs.suricata.io/en/latest/output/eve/eve-json-format.html#event-type-tls) for more details. | TLS log fields | Attribute in OpenTelemetry log | |----------------------------------------|----------------------------------------------------------| diff --git a/extension/encoding/azureencodingextension/README.md b/extension/encoding/azureencodingextension/README.md index 558fd11570d42..a9a2611a411cd 100644 --- a/extension/encoding/azureencodingextension/README.md +++ b/extension/encoding/azureencodingextension/README.md @@ -3,7 +3,6 @@ This extension is designed for unmarshaling logs/traces/metrics encoded in specific format produced by [Azure Diagnostic Settings Export](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/diagnostic-settings) or [Azure Data Collection Rules (DCRs)](https://learn.microsoft.com/en-us/azure/azure-monitor/data-collection/data-collection-rule-overview) - | Status | | | ------------- |-----------| | Stability | [alpha] | diff --git a/extension/encoding/googlecloudlogentryencodingextension/README.md b/extension/encoding/googlecloudlogentryencodingextension/README.md index 865c387ad2f4e..900f776724b18 100644 --- a/extension/encoding/googlecloudlogentryencodingextension/README.md +++ b/extension/encoding/googlecloudlogentryencodingextension/README.md @@ -3,7 +3,6 @@ This extension can be used to unmarshall a [Cloud Logging LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) message type. - | Status | | | ------------- |-----------| | Stability | [alpha] | @@ -83,28 +82,28 @@ The [log entry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEn | `protoPayload` | Placed on the record body as is, unless log type is supported | | `textPayload` | Placed on the record body as is, unless log type is supported | | `jsonPayload` | Placed on the record body as is, unless log type is supported | -| `split.uid` | Log record attribute: `gcp.split.uid` | | -| `split.index` | Log record attribute: `gcp.split.index` | | -| `split.totalSplits` | Log record attribute: `gcp.split.total` | | -| `errorGroups[].id` | Log record attribute: `gcp.error_groups` list
Each element has attribute `id` | | -| `apphub.application.container` | Log record attribute: `gcp.apphub.application.container` | | -| `apphub.application.location` | Log record attribute: `gcp.apphub.application.location` | | -| `apphub.application.id` | Log record attribute: `gcp.apphub.application.id` | | -| `apphub.service.id` | Log record attribute: `gcp.apphub.service.id` | | -| `apphub.service.environmentType` | Log record attribute: `gcp.apphub.service.environment_type` | | -| `apphub.service.criticalityType` | Log record attribute: `gcp.apphub.service.criticality_type` | | -| `apphub.workload.id` | Log record attribute: `gcp.apphub.workload.id` | | -| `apphub.workload.environmentType` | Log record attribute: `gcp.apphub.workload.environment_type` | | -| `apphub.workload.criticalityType` | Log record attribute: `gcp.apphub.workload.criticality_type` | | -| `apphubDestination.application.container` | Log record attribute: `gcp.apphub_destination.application.container` | | -| `apphubDestination.application.location` | Log record attribute: `gcp.apphub_destination.application.location` | | -| `apphubDestination.application.id` | Log record attribute: `gcp.apphub_destination.application.id` | | -| `apphubDestination.service.id` | Log record attribute: `gcp.apphub_destination.service.id` | | -| `apphubDestination.service.environmentType` | Log record attribute: `gcp.apphub_destination.service.environment_type` | | -| `apphubDestination.service.criticalityType` | Log record attribute: `gcp.apphub_destination.service.criticality_type` | | -| `apphubDestination.workload.id` | Log record attribute: `gcp.apphub_destination.workload.id` | | -| `apphubDestination.workload.environmentType` | Log record attribute: `gcp.apphub_destination.workload.environment_type` | | -| `apphubDestination.workload.criticalityType` | Log record attribute: `gcp.apphub_destination.workload.criticality_type` | | +| `split.uid` | Log record attribute: `gcp.split.uid` | +| `split.index` | Log record attribute: `gcp.split.index` | +| `split.totalSplits` | Log record attribute: `gcp.split.total` | +| `errorGroups[].id` | Log record attribute: `gcp.error_groups` list
Each element has attribute `id` | +| `apphub.application.container` | Log record attribute: `gcp.apphub.application.container` | +| `apphub.application.location` | Log record attribute: `gcp.apphub.application.location` | +| `apphub.application.id` | Log record attribute: `gcp.apphub.application.id` | +| `apphub.service.id` | Log record attribute: `gcp.apphub.service.id` | +| `apphub.service.environmentType` | Log record attribute: `gcp.apphub.service.environment_type` | +| `apphub.service.criticalityType` | Log record attribute: `gcp.apphub.service.criticality_type` | +| `apphub.workload.id` | Log record attribute: `gcp.apphub.workload.id` | +| `apphub.workload.environmentType` | Log record attribute: `gcp.apphub.workload.environment_type` | +| `apphub.workload.criticalityType` | Log record attribute: `gcp.apphub.workload.criticality_type` | +| `apphubDestination.application.container` | Log record attribute: `gcp.apphub_destination.application.container` | +| `apphubDestination.application.location` | Log record attribute: `gcp.apphub_destination.application.location` | +| `apphubDestination.application.id` | Log record attribute: `gcp.apphub_destination.application.id` | +| `apphubDestination.service.id` | Log record attribute: `gcp.apphub_destination.service.id` | +| `apphubDestination.service.environmentType` | Log record attribute: `gcp.apphub_destination.service.environment_type` | +| `apphubDestination.service.criticalityType` | Log record attribute: `gcp.apphub_destination.service.criticality_type` | +| `apphubDestination.workload.id` | Log record attribute: `gcp.apphub_destination.workload.id` | +| `apphubDestination.workload.environmentType` | Log record attribute: `gcp.apphub_destination.workload.environment_type` | +| `apphubDestination.workload.criticalityType` | Log record attribute: `gcp.apphub_destination.workload.criticality_type` | #### Severity Mapping @@ -173,7 +172,7 @@ The following format values are supported in the `googlecloudlogentryencodingext | Armor Logs | `armorlog` | Google Cloud armor logs (security policies applied) | | Proxy Network Load Balancer Logs | `proxy-nlb` | Proxy Network Load Balancer connection logs | | Cloud DNS Logs | `dns` | Cloud DNS query and response logs | -| Passthrough Network Load Balancer Logs | `passthrough-nlb` | Passthrough Network Load Balancer flow logs +| Passthrough Network Load Balancer Logs | `passthrough-nlb` | Passthrough Network Load Balancer flow logs | ### Cloud Audit Logs @@ -515,4 +514,3 @@ Passthrough Network Load Balancer flow logs cover both [External](https://cloud. - `1` → `icmp` Resource labels such as `backend_group_name`, `backend_network_name`, `forwarding_rule_name`, and `region` are set with the `gcp.label.*` prefix. - diff --git a/extension/encoding/jsonlogencodingextension/README.md b/extension/encoding/jsonlogencodingextension/README.md index ff46071070fef..977243a864139 100644 --- a/extension/encoding/jsonlogencodingextension/README.md +++ b/extension/encoding/jsonlogencodingextension/README.md @@ -24,7 +24,6 @@ The `body` mode of the JSON encoding extension is used to marshal or unmarshal the JSON log body, ignoring other log fields. - #### body_with_inline_attributes The `body_with_inline_attributes` mode within the JSON encoding extension grabs the resource and attributes and adds them as key value pairs to the JSON body. It iterates through all the logs and creates a JSON array like the following example: diff --git a/extension/encoding/otlpencodingextension/README.md b/extension/encoding/otlpencodingextension/README.md index 8096c71479a47..c6d87c20e385e 100644 --- a/extension/encoding/otlpencodingextension/README.md +++ b/extension/encoding/otlpencodingextension/README.md @@ -3,7 +3,6 @@ This extension unmarshals and marshals data encoded according to the [OTLP specification](https://opentelemetry.io/docs/specs/otlp/). - | Status | | | ------------- |-----------| | Stability | [beta] | diff --git a/extension/encoding/skywalkingencodingextension/README.md b/extension/encoding/skywalkingencodingextension/README.md index 831edd9450903..dfa04317e469e 100644 --- a/extension/encoding/skywalkingencodingextension/README.md +++ b/extension/encoding/skywalkingencodingextension/README.md @@ -3,7 +3,6 @@ The `Skywalking` encoding extension is used to unmarshal Apache Skywalking segment traces. Marshalling is not supported. - | Status | | | ------------- |-----------| | Stability | [alpha] | diff --git a/extension/headerssetterextension/README.md b/extension/headerssetterextension/README.md index f4d40b45a0366..546bcc711c171 100644 --- a/extension/headerssetterextension/README.md +++ b/extension/headerssetterextension/README.md @@ -54,7 +54,7 @@ The following settings are available: The `value`, `value_file`, `from_context,default_value` and `from_attribute,default_value` properties are mutually exclusive. In order for `from_context` to work, other components in the pipeline also need to be configured appropriately: -* If a [batch processor][batch-processor] is present in the pipeline, it must be configured to [preserve client metadata][batch-processor-preserve-metadata]. +* If a [batch processor][batch-processor] is present in the pipeline, it must be configured to [preserve client metadata][batch-processor-preserve-metadata]. Add the value which `from_context` needs to the `metadata_keys` of the batch processor. * Receivers must be configured with `include_metadata: true` so that metadata keys are available to the pipeline. diff --git a/extension/healthcheckextension/README.md b/extension/healthcheckextension/README.md index f636e38854999..93a7b5209d71b 100644 --- a/extension/healthcheckextension/README.md +++ b/extension/healthcheckextension/README.md @@ -29,7 +29,6 @@ > one. See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/11780 for more > details. - Health Check extension enables an HTTP url that can be probed to check the status of the OpenTelemetry Collector. This extension can be used as a liveness and/or readiness probe on Kubernetes. diff --git a/extension/healthcheckv2extension/README.md b/extension/healthcheckv2extension/README.md index c25c8e47987f9..5b9f160628f88 100644 --- a/extension/healthcheckv2extension/README.md +++ b/extension/healthcheckv2extension/README.md @@ -304,7 +304,6 @@ Assuming the health check extension is configured with `http.status.endpoint` se `localhost:13133`, a request to `http://localhost:13133/status?pipeline=traces/http&verbose` will have a response body such as: - ```json { "start_time": "2024-01-18T17:27:12.570394-08:00", diff --git a/extension/httpforwarderextension/README.md b/extension/httpforwarderextension/README.md index 93258d85edd57..bc372c7d0313c 100644 --- a/extension/httpforwarderextension/README.md +++ b/extension/httpforwarderextension/README.md @@ -4,7 +4,6 @@ This extension accepts HTTP requests, optionally adds headers to them and forwards them. The RequestURIs of the original requests are preserved by the extension. - | Status | | | ------------- |-----------| | Stability | [beta] | diff --git a/extension/jaegerremotesampling/README.md b/extension/jaegerremotesampling/README.md index 8907e7a56349d..21f4bb2a6dddd 100644 --- a/extension/jaegerremotesampling/README.md +++ b/extension/jaegerremotesampling/README.md @@ -20,7 +20,6 @@ By default, two listeners are made available: See our [security best practices doc](https://opentelemetry.io/docs/security/config-best-practices/#protect-against-denial-of-service-attacks) to understand how to set the endpoint in different environments. - Note that the port `14250` will clash with the Jaeger Receiver. When both are used, it's recommended to change this extension to use another port. Although this extension is derived from Jaeger, it can be used by any clients who can consume this standard, such as the [OpenTelemetry Java SDK](https://github.com/open-telemetry/opentelemetry-java/tree/v1.9.1/sdk-extensions/jaeger-remote-sampler). @@ -95,4 +94,3 @@ A sampling strategy file could look like: } ``` Source: https://www.jaegertracing.io/docs/1.28/sampling/#collector-sampling-configuration - diff --git a/extension/k8sleaderelector/README.md b/extension/k8sleaderelector/README.md index a5902ff00c845..cf3788d323548 100644 --- a/extension/k8sleaderelector/README.md +++ b/extension/k8sleaderelector/README.md @@ -4,7 +4,6 @@ This extension enables OpenTelemetry components to run in HA mode across a Kubernetes cluster. The component that owns the lease becomes the leader and becomes the active instance. - | Status | | | ------------- |-----------| | Stability | [alpha] | diff --git a/extension/oauth2clientauthextension/testdata/README.md b/extension/oauth2clientauthextension/testdata/README.md index c0c92aaf3170c..ee5475be55dac 100644 --- a/extension/oauth2clientauthextension/testdata/README.md +++ b/extension/oauth2clientauthextension/testdata/README.md @@ -1,4 +1,4 @@ # Test TLS Configuration The test Cert, Key and CA files in this directory have been copied over from [configtls](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/configtls/testdata) -tests. +tests. diff --git a/extension/observer/cfgardenobserver/README.md b/extension/observer/cfgardenobserver/README.md index ce9d20c708554..0b1284fc1da37 100644 --- a/extension/observer/cfgardenobserver/README.md +++ b/extension/observer/cfgardenobserver/README.md @@ -61,7 +61,6 @@ receivers: | cloud_foundry.auth.access_token | string | none | Access Token (auth.type: token) | | cloud_foundry.auth.refresh_token | string | none | Refresh Token (auth.type: token) | - ### Endpoint Variables Endpoint variables exposed by this observer are as follows. diff --git a/extension/observer/dockerobserver/README.md b/extension/observer/dockerobserver/README.md index 1c32b3ece951e..c2660c960a846 100644 --- a/extension/observer/dockerobserver/README.md +++ b/extension/observer/dockerobserver/README.md @@ -21,7 +21,6 @@ Requires Docker API Version 1.44+. The collector will need permissions to access the Docker Engine API, specifically it will need read access to the Docker socket (default `unix:///var/run/docker.sock` on non-Windows and `npipe:////./pipe/docker_engine` on Windows). - ## Example Config ```yaml @@ -106,7 +105,7 @@ alongside any per-port endpoints. This makes every container, including those without exposed ports, discoverable by `receiver_creator` rules of `type == "container"`. The port-less endpoint has no port information (`port` and `alternate_port` are `0`, `transport` is `unknown`). To get exactly -one logging receiver per container, scope with +one logging receiver per container, scope with `rule: type == "container" and port == 0`. default: `false` diff --git a/extension/observer/ecsobserver/README.md b/extension/observer/ecsobserver/README.md index 013685672714f..8b46fadb1db55 100644 --- a/extension/observer/ecsobserver/README.md +++ b/extension/observer/ecsobserver/README.md @@ -338,23 +338,23 @@ Additional information from ECS and EC2. // - FromTargetYAML and ToTargetYAML converts it between prometheus file discovery format in YAML. // - FromTargetJSON and ToTargetJSON converts it between prometheus file discovery format in JSON. type PrometheusECSTarget struct { - Address string `json:"address"` - MetricsPath string `json:"metrics_path"` - Job string `json:"job"` - TaskDefinitionFamily string `json:"task_definition_family"` - TaskDefinitionRevision int `json:"task_definition_revision"` - TaskLaunchType string `json:"task_launch_type"` - TaskGroup string `json:"task_group"` - TaskTags map[string]string `json:"task_tags"` - ContainerName string `json:"container_name"` - ContainerLabels map[string]string `json:"container_labels"` - HealthStatus string `json:"health_status"` - EC2InstanceId string `json:"ec2_instance_id"` - EC2InstanceType string `json:"ec2_instance_type"` - EC2Tags map[string]string `json:"ec2_tags"` - EC2VPCId string `json:"ec2_vpc_id"` - EC2PrivateIP string `json:"ec2_private_ip"` - EC2PublicIP string `json:"ec2_public_ip"` + Address string `json:"address"` + MetricsPath string `json:"metrics_path"` + Job string `json:"job"` + TaskDefinitionFamily string `json:"task_definition_family"` + TaskDefinitionRevision int `json:"task_definition_revision"` + TaskLaunchType string `json:"task_launch_type"` + TaskGroup string `json:"task_group"` + TaskTags map[string]string `json:"task_tags"` + ContainerName string `json:"container_name"` + ContainerLabels map[string]string `json:"container_labels"` + HealthStatus string `json:"health_status"` + EC2InstanceId string `json:"ec2_instance_id"` + EC2InstanceType string `json:"ec2_instance_type"` + EC2Tags map[string]string `json:"ec2_tags"` + EC2VPCId string `json:"ec2_vpc_id"` + EC2PrivateIP string `json:"ec2_private_ip"` + EC2PublicIP string `json:"ec2_public_ip"` } ``` diff --git a/extension/observer/k8sobserver/README.md b/extension/observer/k8sobserver/README.md index 75a11e7b94f7e..0083c62eef500 100644 --- a/extension/observer/k8sobserver/README.md +++ b/extension/observer/k8sobserver/README.md @@ -78,8 +78,8 @@ All fields are optional. |-------------------|-----------|------------------| ---- | | auth_type | string | `serviceAccount` | How to authenticate to the K8s API server. This can be one of `none` (for no auth), `serviceAccount` (to use the standard service account token provided to the agent pod), or `kubeConfig` to use credentials from `~/.kube/config`. | | node | string | | The node name to limit the discovery of pod, port, and node endpoints. Providing no value (the default) results in discovering endpoints for all available nodes. | -| observe_pods | bool | `true` | Whether to report observer pod and port endpoints. If `true` and `node` is specified it will only discover pod and port endpoints whose `spec.nodeName` matches the provided node name. If `true` and `node` isn't specified, it will discover all available pod and port endpoints. Please note that Collector connectivity to pods from other nodes is dependent on your cluster configuration and isn't guaranteed. | -| observe_nodes | bool | `false` | Whether to report observer k8s.node endpoints. If `true` and `node` is specified it will only discover node endpoints whose `metadata.name` matches the provided node name. If `true` and `node` isn't specified, it will discover all available node endpoints. Please note that Collector connectivity to nodes is dependent on your cluster configuration and isn't guaranteed.| +| observe_pods | bool | `true` | Whether to report observer pod and port endpoints. If `true` and `node` is specified it will only discover pod and port endpoints whose `spec.nodeName` matches the provided node name. If `true` and `node` isn't specified, it will discover all available pod and port endpoints. Please note that Collector connectivity to pods from other nodes is dependent on your cluster configuration and isn't guaranteed. | +| observe_nodes | bool | `false` | Whether to report observer k8s.node endpoints. If `true` and `node` is specified it will only discover node endpoints whose `metadata.name` matches the provided node name. If `true` and `node` isn't specified, it will discover all available node endpoints. Please note that Collector connectivity to nodes is dependent on your cluster configuration and isn't guaranteed.| | observe_services | bool | `false` | Whether to report observer k8s.service endpoints.| | observe_ingresses | bool | `false` | Whether to report observer k8s.ingress endpoints.| | namespaces | []string | `[]` | List of namespaces to retrieve resources from. If not set, all namespaces will be observed. Does not apply for nodes, as those are not namespaced resources. | diff --git a/extension/opampcustommessages/README.md b/extension/opampcustommessages/README.md index 8c2db9d162a17..ace05dbd13165 100644 --- a/extension/opampcustommessages/README.md +++ b/extension/opampcustommessages/README.md @@ -8,31 +8,30 @@ This modules contains interfaces and shared code for sending and receiving [cust An extension may implement the `opampcustommessages.CustomCapabilityRegistry` interface, which allows other components to register capabilities to send and receive messages to/from an OpAMP server. For an example of a component implementing this interface, see the [OpAMP extension](../opampextension/README.md). - ### Registering a custom capability Other components may use a configured OpAMP extension to send and receive custom messages to and from an OpAMP server. Components may use the provided `components.Host` from the Start method in order to get a handle to the registry: ```go func Start(_ context.Context, host component.Host) error { - ext, ok := host.GetExtensions()[opampExtensionID] - if !ok { - return fmt.Errorf("extension %q does not exist", opampExtensionID) - } + ext, ok := host.GetExtensions()[opampExtensionID] + if !ok { + return fmt.Errorf("extension %q does not exist", opampExtensionID) + } - registry, ok := ext.(opampcustommessages.CustomCapabilityRegistry) - if !ok { - return fmt.Errorf("extension %q is not a custom message registry", opampExtensionID) - } + registry, ok := ext.(opampcustommessages.CustomCapabilityRegistry) + if !ok { + return fmt.Errorf("extension %q is not a custom message registry", opampExtensionID) + } - handler, err := registry.Register("io.opentelemetry.custom-capability") - if err != nil { - return fmt.Errorf("failed to register custom capability: %w", err) - } + handler, err := registry.Register("io.opentelemetry.custom-capability") + if err != nil { + return fmt.Errorf("failed to register custom capability: %w", err) + } - // ... send/receive messages using the given handler + // ... send/receive messages using the given handler - return nil + return nil } ``` @@ -46,16 +45,16 @@ To send a message, you can use the SendMessage method. Since only one custom mes ```go for { - sendingChan, err := handler.SendMessage("messageType", []byte("message-data")) - switch { - case err == nil: - break - case errors.Is(err, types.ErrCustomMessagePending): - <-sendingChan - continue - default: - return fmt.Errorf("failed to send message: %w", err) - } + sendingChan, err := handler.SendMessage("messageType", []byte("message-data")) + switch { + case err == nil: + break + case errors.Is(err, types.ErrCustomMessagePending): + <-sendingChan + continue + default: + return fmt.Errorf("failed to send message: %w", err) + } } ``` diff --git a/extension/opampextension/README.md b/extension/opampextension/README.md index efd88777f62a8..b5cf71e5552d0 100644 --- a/extension/opampextension/README.md +++ b/extension/opampextension/README.md @@ -72,7 +72,7 @@ See the [opampcustommessages](../opampcustommessages/README.md) module for more ## Using the AcceptsRestartCommand capability to orchestrate collector config updates -If the `accepts_restart_command` capability is enabled (along with the `extension.opampextension.RemoteRestarts` feature gate), upon receiving a restart `ServerToAgentCommand`, the extension will send a `SIGHUP` signal to the Collector process. +If the `accepts_restart_command` capability is enabled (along with the `extension.opampextension.RemoteRestarts` feature gate), upon receiving a restart `ServerToAgentCommand`, the extension will send a `SIGHUP` signal to the Collector process. The `SIGHUP` signal is trapped in the Collector and initiates a graceful restart of all the components by restarting the Collectors service, which restarts all components, pipelines, and Collector self-monitoring. This functionality might be desired if the Collector's config was updated, as the restart also fetches all configuration sources passed to the Collector through `--config` flags passed to the command invocation. diff --git a/extension/pprofextension/README.md b/extension/pprofextension/README.md index ed5a350802fa4..1a5eec64a9452 100644 --- a/extension/pprofextension/README.md +++ b/extension/pprofextension/README.md @@ -45,7 +45,6 @@ extensions: The full list of settings exposed for this exporter are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - ### Go Profiling with pprof basics The profiler can be used to improve a program. @@ -100,4 +99,3 @@ The `list` command is also useful. Type `list ` to see the source code of your function, annotated with the resource consumption (`flat` and `cum` columns like in the `topN` command). If you prefer to view it in your browser, use the `weblist ` command instead. In this view, you can see which line exactly used the most resources and start to improve it. - diff --git a/extension/sigv4authextension/README.md b/extension/sigv4authextension/README.md index 764905af1c973..f56151fd5d068 100644 --- a/extension/sigv4authextension/README.md +++ b/extension/sigv4authextension/README.md @@ -4,7 +4,6 @@ This extension provides Sigv4 authentication for making requests to AWS services. You can read about the [Sigv4 process](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). - | Status | | | ------------- |-----------| | Stability | [beta] | @@ -34,7 +33,6 @@ The configuration fields are as follows: * `service`: **Optional**. The AWS service for AWS Sigv4 * Note for supported services an attempt will be made to obtain a valid service from the endpoint of the service you are exporting to. Supported services include - workspaces, es, logs and traces. - ## Assume Role ### Example Configuration: @@ -70,7 +68,6 @@ service: * The collector must have valid AWS credentials as used by the [AWS SDK for Go](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html#specifying-credentials) - ## Assume Role with Web Identity Configuring `web_identity_token_file` will cause the sigv4auth extension to exchange the token in the specified `web_identity_token_file` for AWS credentials. This is especially useful for authenticating from on-prem systems or other cloud providers via OIDC to publish telemetry to an AWS destination (e.g. Amazon Managed Prometheus). diff --git a/extension/sigv4authextension/design.md b/extension/sigv4authextension/design.md index d874159e020f7..086557d46c275 100644 --- a/extension/sigv4authextension/design.md +++ b/extension/sigv4authextension/design.md @@ -1,33 +1,25 @@ # **Sigv4 Authenticator Extension Design Document** - This document outlines a proposed implementation of a Sigv4 authenticator extension for the OpenTelemetry (OTEL) Collector. The design is based on the OpenTelemetry Collector [configauth](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/configauth) module and also existing authenticator extensions like the [oauth2](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/oauth2clientauthextension) authenticator. - ## **Use Cases** - The Sigv4 authenticator extension provides a way for the OTEL Collector’s Prometheus Remote Write (PRW) Exporter to sign http requests with Sigv4. This would effectively replace the AWS PRW Exporter as the preferred way to send metrics in remote write format to [Amazon Managed Service for Prometheus (AMP)](https://aws.amazon.com/prometheus/). The Sigv4 authenticator extension could be used to sign any http requests that require Sigv4. The Sigv4 authenticator extension will not be able to sign gRPC requests. -

Diagram 1: E2E data path

- ## **Sigv4 Authenticator Implementation Details** - ### **config.go** We first introduce the `config.go` module, which defines the configuration options a user can make and also does some data validation to catch errors. - `Config` is a struct of the AWS Sigv4 authentication configurations. - ```go // Config for the Sigv4 Authenticator @@ -35,11 +27,10 @@ type Config struct { Region string `mapstructure:"region,omitempty"` Service string `mapstructure:"service,omitempty"` AssumeRole AssumeRole `mapstructure:"assume_role"` - credsProvider *aws.CredentialsProvider + credsProvider *aws.CredentialsProvider } ``` - * `Region` is the AWS region for AWS Sigv4. This is an optional field. * Note that an attempt will be made to obtain a valid region from the endpoint of the service you are exporting to * `Service` is the AWS service for AWS Sigv4. This is an optional field. @@ -47,47 +38,39 @@ type Config struct { * `AssumeRole` is the AssumeRole struct that holds the configuration needed to assume a role, which includes the ARN and SessionName * `credsProvider` holds the necessary AWS CredentialsProvider. This is a private field and will not be configured by the user. We store the provider instead of the credentials themselves so we can ensure we have refreshed credentials when we sign a request. -`AssumeRole` is a struct that holds the configuration needed to assume a role. - +`AssumeRole` is a struct that holds the configuration needed to assume a role. ```go type AssumeRole struct { - ARN string `mapstructure:"arn,omitempty"` - SessionName string `mapstructure:"session_name,omitempty"` + ARN string `mapstructure:"arn,omitempty"` + SessionName string `mapstructure:"session_name,omitempty"` } ``` - * `ARN` is the Amazon Resource Name (ARN) of a role to assume. This is an optional field. * `SessionName`: The name of a role session. This is an optional field. - `Validate()` is a method that checks if the Extension configuration is valid. We aim to catch most errors here, so we can ensure that we fail early and to avoid revalidating static data. We also set our AWS credentials here after we check them. - ```go func (cfg *Config) Validate() error { - credsProvider, err := getCredsFromConfig(cfg) - if err != nil { - return fmt.Errorf("could not retrieve credential provider: %w", err) - } - if credsProvider == nil { - return fmt.Errorf("credsProvider cannot be nil") - } - return nil + credsProvider, err := getCredsFromConfig(cfg) + if err != nil { + return fmt.Errorf("could not retrieve credential provider: %w", err) + } + if credsProvider == nil { + return fmt.Errorf("credsProvider cannot be nil") + } + return nil } ``` - ### **extension.go** - Next, we introduce `extension.go`, which contains the bulk of the implementation of the Sigv4 authenticator. This includes implementing the `ClientAuthenticator` interface and its necessary methods. - `sigv4Auth` is a struct that implements the `ClientAuthenticator` interface. It must implement two methods, `RoundTripper()` and `PerRPCCredentials()`. Additionally, it must also implement a `Start()` and a `Shutdown()` method. For our Sigv4 authenticator, both of these methods won’t do anything, so we instead embed `componenthelper.StartFunc` and `componenthelper.ShutdownFunc` to get that default behavior. - ```go type sigv4Auth struct { cfg *Config @@ -98,9 +81,7 @@ type sigv4Auth struct { } ``` - -`RoundTripper()` is a method that returns a custom `signingRoundTripper`. We use the `sigv4Auth` struct here instead of the `signingRoundTripper` struct for `RoundTripper()` because `sigv4Auth` is the struct that implements the `ClientAuthenticator` interface. Additionally, it makes it so that `RoundTripper()` creates a new round tripper every time for concurrency purposes. - +`RoundTripper()` is a method that returns a custom `signingRoundTripper`. We use the `sigv4Auth` struct here instead of the `signingRoundTripper` struct for `RoundTripper()` because `sigv4Auth` is the struct that implements the `ClientAuthenticator` interface. Additionally, it makes it so that `RoundTripper()` creates a new round tripper every time for concurrency purposes. ```go func (sa *sigv4Auth) RoundTripper(base http.RoundTripper) (http.RoundTripper, error) @@ -120,20 +101,16 @@ func (sa *sigv4Auth) RoundTripper(base http.RoundTripper) (http.RoundTripper, er } ``` - `PerRPCCredentials()` will be implemented to satisfy the `ClientAuthenticator` interface, but AWS API requests don’t use gRPC, so the Sigv4 authenticator will not implement this method. Instead, it will return nil and an error stating that it is not implemented. - ```go func (sa *sigv4Auth) PerRPCCredentials() (credentials.PerRPCCredentials, error) { return nil, errors.New("Not Implemented") } ``` - `newSigv4Extension()` is a function called by `createExtension()` in `factory.go`. It returns a new `sigv4Auth` struct. - ```go func newSigv4Extension(cfg *Config, awsSDKInfo string, logger *zap.logger) (*sigv4Auth, error) { return &sigv4Auth{ @@ -144,69 +121,62 @@ func newSigv4Extension(cfg *Config, awsSDKInfo string, logger *zap.logger) (*sig } ``` - `getCredsFromConfig()` is a function that gets AWS credentials from the Config. It returns `*aws.Credentials`. - ```go func getCredsFromConfig(cfg *Config) (*aws.Credentials, error) { - awscfg, err := awsconfig.LoadDefaultConfig(context.Background(), - awsconfig.WithRegion(cfg.Region), - ) - if err != nil { - return nil, err - } - if cfg.RoleARN != "" { - stsSvc := sts.NewFromConfig(awscfg) - - identifier := cfg.RoleSessionName - if identifier == "" { - b := make([]byte, 5) - rand.Read(b) - identifier = base32.StdEncoding.EncodeToString(b) - } - - provider := stscreds.NewAssumeRoleProvider(stsSvc, cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { - o.RoleSessionName = "otel-" + identifier - }) - awscfg.Credentials = aws.NewCredentialsCache(provider) - } - - _, err = awscfg.Credentials.Retrieve(context.Background()) - if err != nil { - return nil, err - } - - return &awscfg.Credentials, nil + awscfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion(cfg.Region), + ) + if err != nil { + return nil, err + } + if cfg.RoleARN != "" { + stsSvc := sts.NewFromConfig(awscfg) + + identifier := cfg.RoleSessionName + if identifier == "" { + b := make([]byte, 5) + rand.Read(b) + identifier = base32.StdEncoding.EncodeToString(b) + } + + provider := stscreds.NewAssumeRoleProvider(stsSvc, cfg.RoleARN, func(o *stscreds.AssumeRoleOptions) { + o.RoleSessionName = "otel-" + identifier + }) + awscfg.Credentials = aws.NewCredentialsCache(provider) + } + + _, err = awscfg.Credentials.Retrieve(context.Background()) + if err != nil { + return nil, err + } + + return &awscfg.Credentials, nil } ``` - `cloneRequest()` is a function that clones an `http.request` for thread safety purposes. - ```go func cloneRequest(r *http.Request) *http.Request { - // shallow copy of the struct - r2 := new(http.Request) - *r2 = *r - // deep copy of the Header - r2.Header = make(http.Header, len(r.Header)) - for k, s := range r.Header { - r2.Header[k] = append([]string(nil), s...) - } - return r2 + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 } ``` - ### **signingroundtripper.go** `signingroundtripper.go` contains the rest of the implementation of the Sigv4 authenticator. We implement the `RoundTripper` interface and its necessary methods. - `signingRoundTripper` is a custom `RoundTripper` struct (i.e. a custom `http.RoundTripper`). This struct implements the [RoundTripper interface](https://pkg.go.dev/net/http#RoundTripper.RoundTrip) and will implement the method `RoundTrip()`. - ```go type signingRoundTripper struct { transport http.RoundTripper @@ -215,14 +185,12 @@ type signingRoundTripper struct { service string credsProvider *aws.CredentialsProvider awsSDKInfo string - logger *zap.Logger + logger *zap.Logger } ``` - `RoundTrip()` executes a single HTTP transaction and returns an HTTP response. It will sign the request with Sigv4. This method is implemented to satisfy the `RoundTripper` interface. Here, we will have multiple error checks throughout the method to ensure a proper `RoundTrip()` will succeed. - ```go func (si *signingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { reqBody, err := req.GetBody() @@ -250,34 +218,31 @@ func (si *signingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err } req2.Header.Set("User-Agent", ua) - // Hash the request - h := sha256.New() - _, _ = io.Copy(h, body) - payloadHash := hex.EncodeToString(h.Sum(nil)) - - // Use user provided service/region if specified, use inferred service/region if not, then sign the request - service, region := si.inferServiceAndRegion(req) - creds, err := (*si.credsProvider).Retrieve(req.Context()) - if err != nil { - return nil, fmt.Errorf("error retrieving credentials: %w", err) - } - err = si.signer.SignHTTP(req.Context(), creds, req2, payloadHash, service, region, time.Now()) - if err != nil { - return nil, fmt.Errorf("error signing the request: %w", err) - } - - // Send the request - return si.transport.RoundTrip(req2) + // Hash the request + h := sha256.New() + _, _ = io.Copy(h, body) + payloadHash := hex.EncodeToString(h.Sum(nil)) + + // Use user provided service/region if specified, use inferred service/region if not, then sign the request + service, region := si.inferServiceAndRegion(req) + creds, err := (*si.credsProvider).Retrieve(req.Context()) + if err != nil { + return nil, fmt.Errorf("error retrieving credentials: %w", err) + } + err = si.signer.SignHTTP(req.Context(), creds, req2, payloadHash, service, region, time.Now()) + if err != nil { + return nil, fmt.Errorf("error signing the request: %w", err) + } + + // Send the request + return si.transport.RoundTrip(req2) } ``` - **Performance Considerations of `RoundTrip()`** - We take a closer look at the performance of `RoundTrip()`, since it will be heavily used i.e. called by every HTTP transaction, and can also be used for exporting to any AWS service. Our goal here is to outline both the memory and runtime performance of this method, as well as explain design choices. - ```go reqBody, err := req.GetBody() . @@ -291,13 +256,10 @@ We take a closer look at the performance of `RoundTrip()`, since it will be heav body := bytes.NewReader(content) ``` - First, we obtain the body. This should not have an impact on runtime performance or memory performance as we do not expect to handle http requests that are exponentially large in size*. `body` is necessary as it is passed into the `Sign()` method later. - **Note that OTEL does not limit the size of HTTP requests, except if the exporters themselves set a limit. No information could be found on the size of AWS API requests either. If the request body is exponentially large it could affect performance but it is a reasonably safe assumption that will not be the case.* - ```go req2 := cloneRequest(req) . @@ -308,49 +270,37 @@ First, we obtain the body. This should not have an impact on runtime performance req2.Header.Set("User-Agent", ua) ``` - Next, we clone the request and also add runtime information to the User-Agent header. `cloneRequest()` is a helper function that makes a shallow copy of the request and a deep copy of the header. While this may slow the performance of `RoundTrip()`, it is necessary to ensure thread safety*. Overall, this should not have a large impact on either runtime or memory performance, as again we do not expect to handle http requests that are exponentially large in size. Adding runtime information to the header also will not impact memory or runtime performance either. `req2` will be passed into `Sign()`. - **Design Decision: The necessity of `cloneRequest()` is due to the necessity to read the body. If we want to avoid using `cloneRequest()`, that can only be done by avoiding reading the body.* - - - ```go h := sha256.New() - _, _ = io.Copy(h, body) - payloadHash := hex.EncodeToString(h.Sum(nil)) + _, _ = io.Copy(h, body) + payloadHash := hex.EncodeToString(h.Sum(nil)) err = si.signer.SignHTTP(req.Context(), creds, req2, payloadHash, service, region, time.Now()) resp, err := si.transport.RoundTrip(req2) return resp, nil ``` - Lastly, we compute the hash, sign the request, send the request, and return the response. These are all necessary for the authenticator, and while this is the main point of consideration for the performance of `RoundTrip()`, it cannot be optimized further. - `inferServiceAndRegionFromRequestURL()` attempts to infer a region and a service from the URL in the `http.request`. - ```go func (si *signingRoundTripper) inferServiceAndRegionFromRequestURL(r *http.Request) (service string, region string) { // check for service and region from URL - return service, region + return service, region } ``` - ### **factory.go** - Lastly, we introduce `factory.go`, which provides the logic for creating the Sigv4 authenticator extension. - -`NewFactory()` creates a factory for the Sigv4 Authenticator Extension. - +`NewFactory()` creates a factory for the Sigv4 Authenticator Extension. ```go func NewFactory() extension.Factory { @@ -361,20 +311,16 @@ func NewFactory() extension.Factory { } ``` - `createDefaultConfig()` usually creates a config struct with default values. In our case, there are no sensible defaults to provide so it won’t do much except set the ID. - ```go func createDefaultConfig() component.Config { return &Config{} } ``` - `createExtension()` calls `newSigv4Extension()` in `extension.go` to create the extension. Here, we add `awsSDKInfo` that will be used later to add that info to our `signingRoundTripper` in our `RoundTripper()` method. - ```go func createExtension(_ context.Context, set extension.CreateSettings, cfg component.Config) (extension.Extension, error) { awsSDKInfo := fmt.Sprintf("%s/%s", aws.SDKName, aws.SDKVersion) @@ -382,52 +328,40 @@ func createExtension(_ context.Context, set extension.CreateSettings, cfg compon } ``` - ### **Sigv4 Component Diagram** - Below is a diagram of the Sigv4 authenticator’s components and how the interact with each other. This is a detailed look at the “Sigv4 Authenticator” box in diagram 1. -

Diagram 2: Sigv4 component diagram

- ## **Test Design** - ### **config_test.go** - `TestLoadConfig()` - ```go func TestLoadConfig(t *testing.T) { //Load config yaml from test data and assert against expected values } ``` - `TestLoadConfigError()` - ```go func TestLoadConfigError(t *testing.T) { // Load bad config yaml from test data and assert right errors thrown } ``` - ### **extension_test.go** - `TestNewSigv4Extension()` - ```go func TestNewSigv4Extension(t *testing.T) { // assert that newSigv4Extension() returns a non nil, correct sigv4Auth @@ -435,10 +369,8 @@ func TestNewSigv4Extension(t *testing.T) { } ``` - `TestRoundTripper()` - ```go func TestRoundTripper(t *testing.T) { // assert that a signingRoundTripper created with region/service @@ -448,14 +380,12 @@ func TestRoundTripper(t *testing.T) { } ``` - ```go func TestPerRPCCredentials(t *testing.T) { // assert that an error is thrown } ``` - ```go func TestGetCredsFromConfig(t *testing.T) { // assert that credentials can be fetched properly and no error is thrown @@ -464,20 +394,16 @@ func TestGetCredsFromConfig(t *testing.T) { } ``` - ```go func TestCloneRequest(t *testing.T) { // assert that a request and a clone have equal headers and body } ``` - ### **signingroundtripper_test.go** - `TestRoundTrip()` - ```go func TestRoundTrip(t *testing.T) { // assert http req are signed by sigv4 @@ -486,7 +412,6 @@ func TestRoundTrip(t *testing.T) { } ``` - ```go func TestInferServiceAndRegionFromRequestURL(t *testing.T) { // assert empty service/region string if invalid URL @@ -494,46 +419,36 @@ func TestInferServiceAndRegionFromRequestURL(t *testing.T) { } ``` - ### **factory_test.go** - `TestNewFactory()` - ```go func TestNewFactory(t *testing.T) { // assert that the new factory is not nil } ``` - `TestCreateDefaultConfig()` - ```go func TestCreateDefaultConfig(t *testing.T) { // assert createDefaultConfig() creates config struct that only sets ID } ``` - `TestCreateExtension()` - ```go func TestCreateExtension(t *testing.T) { // assert no error/nil } ``` - ## **Package File Structure** - File structure will mimic that of existing authentication extension implementations. - * In `opentelemetry-collector-contrib/extension`, the `sigv4authextension` directory will be added, containing the `sigv4authextension` package. Within this directory will be: * `config.go` * `extension.go` @@ -554,11 +469,9 @@ File structure will mimic that of existing authentication extension implementati * `config.yaml` to load a good config and test against expected values * `config_bad.yaml`to load a bad config and test that the right errors are thrown - Additionally, a few other files will be modified to reflect changes made, for example: - -* `CHANGELOG.md` +* `CHANGELOG.md` * `go.mod` * `go.sum` * `versions.yaml` diff --git a/extension/storage/filestorage/README.md b/extension/storage/filestorage/README.md index 49c51c426d0b0..a928658b40133 100644 --- a/extension/storage/filestorage/README.md +++ b/extension/storage/filestorage/README.md @@ -21,7 +21,7 @@ The File Storage extension can persist state to the local file system. The extension requires read and write access to a directory. A default directory can be used, but it must already exist in order for the extension to operate. -`directory` is the relative or absolute path to the dedicated data storage directory. +`directory` is the relative or absolute path to the dedicated data storage directory. The default directory is `%ProgramData%\Otelcol\FileStorage` on Windows and `/var/lib/otelcol/file_storage` otherwise. `timeout` is the maximum time to wait for a file lock. This value does not need to be modified in most circumstances. @@ -33,8 +33,8 @@ The default timeout is `1s`. By default, the directories will be created with `0750 (rwxr-x---)` permissions, minus the process umask. Use `directory_permissions` to customize directory creation permissions, minus the process umask. -`recreate` when set, the filestorage extension will automatically rename the corrupted bbolt database and create a new one when certain bbolt panics occur. -See (#35899)[https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35899] for more details. +`recreate` when set, the filestorage extension will automatically rename the corrupted bbolt database and create a new one when certain bbolt panics occur. +See [#35899](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35899) for more details. If the database fails to open due to corruption (resulting in a panic), the corrupted file will be automatically renamed to `{filename}.{ISO 8601 timestamp}.backup` and a new data file will be created from scratch. This allows the collector to continue operating even when encountering certain bbolt panics. If no corruption is detected, the existing database continues to be used normally. There may still be scenarios where manually removing or renaming the file may be required, and this feature flag is not a panacea for all bbolt panics you can encounter. diff --git a/internal/otelarrow/admission2/README.md b/internal/otelarrow/admission2/README.md index 9076096e2ec58..8cbffda5cb005 100644 --- a/internal/otelarrow/admission2/README.md +++ b/internal/otelarrow/admission2/README.md @@ -16,7 +16,7 @@ The BoundedQueue implements LIFO semantics. See this [article](https://medium.com/swlh/fifo-considered-harmful-793b76f98374) explaining why it is preferred to FIFO semantics. -## Usage +## Usage Create a new BoundedQueue by calling `bq := admission.NewBoundedQueue(maxLimitBytes, maxLimitWaiting)` diff --git a/internal/otelarrow/netstats/README.md b/internal/otelarrow/netstats/README.md index 55783ec7d2bfe..58911b332d972 100644 --- a/internal/otelarrow/netstats/README.md +++ b/internal/otelarrow/netstats/README.md @@ -15,7 +15,7 @@ to `netstats.NewExporterNetworkReporter` or `netstats.NewExporterNetworkReporter`, then register with gRPC: ``` - dialOpts = append(dialOpts, grpc.WithStatsHandler(netReporter.Handler())) + dialOpts = append(dialOpts, grpc.WithStatsHandler(netReporter.Handler())) ``` Because OTel-Arrow supports the use of compressed payloads, configured @@ -27,10 +27,10 @@ compressed and uncompressed sizes. To report only uncompressed size in the exporter case, for example: ``` - var sized netstats.SizesStruct - sized.Method = s.method - sized.Length = int64(uncompressedSize) - netReporter.CountSend(ctx, sized) + var sized netstats.SizesStruct + sized.Method = s.method + sized.Length = int64(uncompressedSize) + netReporter.CountSend(ctx, sized) ``` Likewise, the receiver uses `CountRecv` with `sized.Length` set to diff --git a/issue-triaging.md b/issue-triaging.md index 526d035f6ccb5..74ebe0912aab0 100644 --- a/issue-triaging.md +++ b/issue-triaging.md @@ -83,7 +83,7 @@ It is recommended that a triager should follow these steps when a new issue is r - The enhancement/bugfix has already been addressed in the newer version. - The new component has been added to the repository. -#### Triage process flowchart +#### Triage process flowchart ```mermaid flowchart TD diff --git a/pkg/datadog/agentcomponents/agentcomponents_options.md b/pkg/datadog/agentcomponents/agentcomponents_options.md index ef63970583bbd..6ddf948aa1c1c 100644 --- a/pkg/datadog/agentcomponents/agentcomponents_options.md +++ b/pkg/datadog/agentcomponents/agentcomponents_options.md @@ -4,130 +4,130 @@ This file provides examples of how to use the NewConfigComponent options pattern package agentcomponents import ( - pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/configopaque" + pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configopaque" - datadogconfig "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/datadog/config" + datadogconfig "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/datadog/config" ) // Example 1: A metrics-only module that only needs API configuration func exampleMetricsOnlyModule(_ component.TelemetrySettings, apiKey configopaque.String) { - config := NewConfigComponent( - WithAPIConfig(&datadogconfig.Config{ - API: datadogconfig.APIConfig{ - Key: apiKey, - Site: "datadoghq.com", - }, - }), - WithCustomConfig("metrics.enabled", true, pkgconfigmodel.SourceFile), - WithCustomConfig("metrics.batch_size", 100, pkgconfigmodel.SourceDefault), - ) - _ = config // Use the config component for metrics setup + config := NewConfigComponent( + WithAPIConfig(&datadogconfig.Config{ + API: datadogconfig.APIConfig{ + Key: apiKey, + Site: "datadoghq.com", + }, + }), + WithCustomConfig("metrics.enabled", true, pkgconfigmodel.SourceFile), + WithCustomConfig("metrics.batch_size", 100, pkgconfigmodel.SourceDefault), + ) + _ = config // Use the config component for metrics setup } // Example 2: A traces module with custom configuration func exampleTracesModule(set component.TelemetrySettings, cfg *datadogconfig.Config) { - config := NewConfigComponent( - WithAPIConfig(cfg), - WithLogLevel(set), - WithCustomConfig("traces.enabled", true, pkgconfigmodel.SourceFile), - WithCustomConfig("traces.sample_rate", 0.1, pkgconfigmodel.SourceDefault), - WithCustomConfig("traces.max_traces_per_second", 1000, pkgconfigmodel.SourceDefault), - WithProxyFromEnv(), - ) - _ = config // Use the config component for traces setup + config := NewConfigComponent( + WithAPIConfig(cfg), + WithLogLevel(set), + WithCustomConfig("traces.enabled", true, pkgconfigmodel.SourceFile), + WithCustomConfig("traces.sample_rate", 0.1, pkgconfigmodel.SourceDefault), + WithCustomConfig("traces.max_traces_per_second", 1000, pkgconfigmodel.SourceDefault), + WithProxyFromEnv(), + ) + _ = config // Use the config component for traces setup } // Example 3: A full-featured logs module using all logs options func exampleLogsModule(set component.TelemetrySettings, cfg *datadogconfig.Config) { - config := NewConfigComponent( - WithAPIConfig(cfg), - WithLogLevel(set), - WithLogsConfig(cfg), - WithLogsDefaults(), - WithProxyFromEnv(), - // Additional logs-specific configuration - WithCustomConfig("logs.dev_mode", false, pkgconfigmodel.SourceDefault), - WithCustomConfig("logs.extra_metadata", true, pkgconfigmodel.SourceFile), - ) - _ = config // Use the config component for logs setup + config := NewConfigComponent( + WithAPIConfig(cfg), + WithLogLevel(set), + WithLogsConfig(cfg), + WithLogsDefaults(), + WithProxyFromEnv(), + // Additional logs-specific configuration + WithCustomConfig("logs.dev_mode", false, pkgconfigmodel.SourceDefault), + WithCustomConfig("logs.extra_metadata", true, pkgconfigmodel.SourceFile), + ) + _ = config // Use the config component for logs setup } // Example 4: A minimal configuration for a custom connector/processor func exampleMinimalModule(_ component.TelemetrySettings, apiKey configopaque.String) { - config := NewConfigComponent( - WithAPIConfig(&datadogconfig.Config{ - API: datadogconfig.APIConfig{ - Key: apiKey, - Site: "datadoghq.eu", // Different site - }, - }), - // Only set what's needed for this module - WithCustomConfig("connector.buffer_size", 500, pkgconfigmodel.SourceDefault), - ) - _ = config // Use the config component for connector setup + config := NewConfigComponent( + WithAPIConfig(&datadogconfig.Config{ + API: datadogconfig.APIConfig{ + Key: apiKey, + Site: "datadoghq.eu", // Different site + }, + }), + // Only set what's needed for this module + WithCustomConfig("connector.buffer_size", 500, pkgconfigmodel.SourceDefault), + ) + _ = config // Use the config component for connector setup } // Example 5: Advanced usage with conditional options func exampleConditionalModule(set component.TelemetrySettings, cfg *datadogconfig.Config, enableAdvanced bool) { - options := []ConfigOption{ - WithAPIConfig(cfg), - WithLogLevel(set), - } - - // Conditionally add more options based on configuration - if enableAdvanced { - options = append(options, - WithLogsDefaults(), - WithProxyFromEnv(), - WithCustomConfig("advanced.feature_flag", true, pkgconfigmodel.SourceFile), - ) - } - - // Add development-specific options in dev environments - if cfg.API.Site == "datadoghq.local" { - options = append(options, - WithCustomConfig("dev.debug_mode", true, pkgconfigmodel.SourceDefault), - WithCustomConfig("dev.verbose_logging", true, pkgconfigmodel.SourceDefault), - ) - } - - config := NewConfigComponent(options...) - _ = config // Use the config component + options := []ConfigOption{ + WithAPIConfig(cfg), + WithLogLevel(set), + } + + // Conditionally add more options based on configuration + if enableAdvanced { + options = append(options, + WithLogsDefaults(), + WithProxyFromEnv(), + WithCustomConfig("advanced.feature_flag", true, pkgconfigmodel.SourceFile), + ) + } + + // Add development-specific options in dev environments + if cfg.API.Site == "datadoghq.local" { + options = append(options, + WithCustomConfig("dev.debug_mode", true, pkgconfigmodel.SourceDefault), + WithCustomConfig("dev.verbose_logging", true, pkgconfigmodel.SourceDefault), + ) + } + + config := NewConfigComponent(options...) + _ = config // Use the config component } // Example 6: Creating reusable option sets for common patterns var ( - // Common options for production environments - ProductionOptions = []ConfigOption{ - WithLogsDefaults(), - WithProxyFromEnv(), - } - - // Common options for development environments - DevelopmentOptions = []ConfigOption{ - WithCustomConfig("dev.debug_mode", true, pkgconfigmodel.SourceDefault), - WithCustomConfig("dev.mock_endpoints", true, pkgconfigmodel.SourceDefault), - } + // Common options for production environments + ProductionOptions = []ConfigOption{ + WithLogsDefaults(), + WithProxyFromEnv(), + } + + // Common options for development environments + DevelopmentOptions = []ConfigOption{ + WithCustomConfig("dev.debug_mode", true, pkgconfigmodel.SourceDefault), + WithCustomConfig("dev.mock_endpoints", true, pkgconfigmodel.SourceDefault), + } ) func exampleReusableOptions(set component.TelemetrySettings, cfg *datadogconfig.Config, isProd bool) { - baseOptions := []ConfigOption{ - WithAPIConfig(cfg), - WithLogLevel(set), - } - - var envOptions []ConfigOption - if isProd { - envOptions = ProductionOptions - } else { - envOptions = DevelopmentOptions - } - - // Combine base options with environment-specific options - allOptions := append(baseOptions, envOptions...) - config := NewConfigComponent(allOptions...) - _ = config // Use the config component + baseOptions := []ConfigOption{ + WithAPIConfig(cfg), + WithLogLevel(set), + } + + var envOptions []ConfigOption + if isProd { + envOptions = ProductionOptions + } else { + envOptions = DevelopmentOptions + } + + // Combine base options with environment-specific options + allOptions := append(baseOptions, envOptions...) + config := NewConfigComponent(allOptions...) + _ = config // Use the config component } ``` diff --git a/pkg/expohisto/README.md b/pkg/expohisto/README.md index 12e624ec925b7..dc223403f843f 100644 --- a/pkg/expohisto/README.md +++ b/pkg/expohisto/README.md @@ -23,11 +23,11 @@ greater difference between minimum and maximum bucket index: ```golang func bucketsNeeded(minValue, maxValue float64, scale int32) int32 { - return bucketIndex(maxValue, scale) - bucketIndex(minValue, scale) + 1 + return bucketIndex(maxValue, scale) - bucketIndex(minValue, scale) + 1 } func bucketIndex(value float64, scale int32) int32 { - return math.Log(value) * math.Ldexp(math.Log2E, scale) + return math.Log(value) * math.Ldexp(math.Log2E, scale) } ``` @@ -73,10 +73,10 @@ maximum size. The structure of a single range of buckets is: ```golang type buckets struct { - backing bucketsVarwidth[T] // for T = uint8 | uint16 | uint32 | uint64 - indexBase int32 - indexStart int32 - indexEnd int32 + backing bucketsVarwidth[T] // for T = uint8 | uint16 | uint32 | uint64 + indexBase int32 + indexStart int32 + indexEnd int32 } ``` @@ -128,11 +128,11 @@ func newScale(minIndex, maxIndex, scale, maxSize int32) int32 { func changeScale(minIndex, maxIndex, scale, maxSize int32) int32 { var change int32 for maxIndex - minIndex >= maxSize { - maxIndex >>= 1 - minIndex >>= 1 - change++ + maxIndex >>= 1 + minIndex >>= 1 + change++ } - return change + return change } ``` diff --git a/pkg/golden/README.md b/pkg/golden/README.md index 9b27d42facf9f..7bf290d5238ac 100644 --- a/pkg/golden/README.md +++ b/pkg/golden/README.md @@ -10,7 +10,7 @@ [alpha]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#alpha -The package golden provides utilities for reading and writing files with metrics, traces and logs in YAML format. +The package golden provides utilities for reading and writing files with metrics, traces and logs in YAML format. The package is expected to be used with pkg/pdatatest module. ## Generating an expected result file @@ -31,24 +31,24 @@ necessary to ensure the function is removed after the golden file is written. ```go func TestScraper(t *testing.T) { - cfg := createDefaultConfig().(*Config) - require.NoError(t, xconfmap.Validate(cfg)) + cfg := createDefaultConfig().(*Config) + require.NoError(t, xconfmap.Validate(cfg)) - scraper := newScraper(componenttest.NewNopReceiverCreateSettings(), cfg) + scraper := newScraper(componenttest.NewNopReceiverCreateSettings(), cfg) - err := scraper.start(context.Background(), componenttest.NewNopHost()) - require.NoError(t, err) + err := scraper.start(context.Background(), componenttest.NewNopHost()) + require.NoError(t, err) - actualMetrics, err := scraper.scrape(context.Background()) - require.NoError(t, err) + actualMetrics, err := scraper.scrape(context.Background()) + require.NoError(t, err) - expectedFile := filepath.Join("testdata", "scraper", "expected.yaml") + expectedFile := filepath.Join("testdata", "scraper", "expected.yaml") - golden.WriteMetrics(t, expectedFile, actualMetrics) // This line is temporary! TODO remove this!! + golden.WriteMetrics(t, expectedFile, actualMetrics) // This line is temporary! TODO remove this!! - expectedMetrics, err := golden.ReadMetrics(expectedFile) - require.NoError(t, err) + expectedMetrics, err := golden.ReadMetrics(expectedFile) + require.NoError(t, err) - require.NoError(t, pmetrictest.CompareMetrics(expectedMetrics, actualMetrics)) + require.NoError(t, pmetrictest.CompareMetrics(expectedMetrics, actualMetrics)) } ``` diff --git a/pkg/ottl/LANGUAGE.md b/pkg/ottl/LANGUAGE.md index 2e9c49d29c723..38aaf973b397c 100644 --- a/pkg/ottl/LANGUAGE.md +++ b/pkg/ottl/LANGUAGE.md @@ -219,14 +219,13 @@ Division by zero is gracefully handled with an error, but other arithmetic opera Division of integers results in an integer and follows Go's rules for division of integers. Since Math Expressions support `Path`s and `Converter`s as input, they are evaluated during data processing. -__As a result, in order for a function to be able to accept an Math Expressions as a parameter it must use a `Getter`.__ +**As a result, in order for a function to be able to accept an Math Expressions as a parameter it must use a `Getter`.** Example Math Expressions - `1 + 1` - `end_time_unix_nano - end_time_unix_nano` - `sum([1, 2, 3, 4]) + (10 / 1) - 1` - ### Boolean Expressions Boolean Expressions allow a decision to be made about whether an Editor should be called. Boolean Expressions are optional. When used, the parsed statement will include a `Condition`, which can be used to evaluate the result of the statement's Boolean Expression. Boolean Expressions always evaluate to a boolean value (true or false). diff --git a/pkg/ottl/README.md b/pkg/ottl/README.md index 45582ea888882..1d35ea12c2771 100644 --- a/pkg/ottl/README.md +++ b/pkg/ottl/README.md @@ -21,7 +21,6 @@ This package implements everything necessary to use OTTL in a Collector componen - [Troubleshooting](#troubleshooting) - [Resources](#resources) - ## Getting Started An OTTL statement is made up of 2 parts: diff --git a/pkg/ottl/contexts/README.md b/pkg/ottl/contexts/README.md index 8a86518d39709..c077c953e7dec 100644 --- a/pkg/ottl/contexts/README.md +++ b/pkg/ottl/contexts/README.md @@ -8,4 +8,4 @@ A Context's `PathExpressionParser` is what the OTTL will use to interpret a Path A Context's `EnumParser` is what the OTTL will use to interpret an Enum Symbol. For the data model being represented, it should be able to handle any incoming Enum Symbol and return the appropriate Enum value. It should return an error if the Enum Symbol is not known. -Context implementations for Traces, Metrics, and Logs are provided by this module. It is recommended to use these contexts when using the OTTL to interact with OpenTelemetry traces, metrics, and logs. \ No newline at end of file +Context implementations for Traces, Metrics, and Logs are provided by this module. It is recommended to use these contexts when using the OTTL to interact with OpenTelemetry traces, metrics, and logs. \ No newline at end of file diff --git a/pkg/ottl/contexts/ottldatapoint/README.md b/pkg/ottl/contexts/ottldatapoint/README.md index 7f2191f09e97c..7a6fbcbbaba7e 100644 --- a/pkg/ottl/contexts/ottldatapoint/README.md +++ b/pkg/ottl/contexts/ottldatapoint/README.md @@ -53,7 +53,7 @@ The following paths are supported. ## Enums -The DataPoint Context supports the enum names from the [metrics proto](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto). +The DataPoint Context supports the enum names from the [metrics proto](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto). In addition, it also supports an enum for metrics data type, with the numeric value being [defined by pdata](https://github.com/open-telemetry/opentelemetry-collector/blob/main/pdata/pmetric/metrics.go). diff --git a/pkg/ottl/contexts/ottllog/README.md b/pkg/ottl/contexts/ottllog/README.md index 2f099f71cabb9..cfde4c7f6b278 100644 --- a/pkg/ottl/contexts/ottllog/README.md +++ b/pkg/ottl/contexts/ottllog/README.md @@ -55,26 +55,26 @@ The Log Context supports the enum names from the [logs proto](https://github.com |-----------------------------|-------| | SEVERITY_NUMBER_UNSPECIFIED | 0 | | SEVERITY_NUMBER_TRACE | 1 | -| SEVERITY_NUMBER_TRACE2 | 2 | -| SEVERITY_NUMBER_TRACE3 | 3 | -| SEVERITY_NUMBER_TRACE4 | 4 | -| SEVERITY_NUMBER_DEBUG | 5 | -| SEVERITY_NUMBER_DEBUG2 | 6 | -| SEVERITY_NUMBER_DEBUG3 | 7 | -| SEVERITY_NUMBER_DEBUG4 | 8 | -| SEVERITY_NUMBER_INFO | 9 | -| SEVERITY_NUMBER_INFO2 | 10 | -| SEVERITY_NUMBER_INFO3 | 11 | -| SEVERITY_NUMBER_INFO4 | 12 | -| SEVERITY_NUMBER_WARN | 13 | -| SEVERITY_NUMBER_WARN2 | 14 | -| SEVERITY_NUMBER_WARN3 | 15 | -| SEVERITY_NUMBER_WARN4 | 16 | -| SEVERITY_NUMBER_ERROR | 17 | -| SEVERITY_NUMBER_ERROR2 | 18 | -| SEVERITY_NUMBER_ERROR3 | 19 | -| SEVERITY_NUMBER_ERROR4 | 20 | -| SEVERITY_NUMBER_FATAL | 21 | -| SEVERITY_NUMBER_FATAL2 | 22 | -| SEVERITY_NUMBER_FATAL3 | 23 | -| SEVERITY_NUMBER_FATAL4 | 24 | \ No newline at end of file +| SEVERITY_NUMBER_TRACE2 | 2 | +| SEVERITY_NUMBER_TRACE3 | 3 | +| SEVERITY_NUMBER_TRACE4 | 4 | +| SEVERITY_NUMBER_DEBUG | 5 | +| SEVERITY_NUMBER_DEBUG2 | 6 | +| SEVERITY_NUMBER_DEBUG3 | 7 | +| SEVERITY_NUMBER_DEBUG4 | 8 | +| SEVERITY_NUMBER_INFO | 9 | +| SEVERITY_NUMBER_INFO2 | 10 | +| SEVERITY_NUMBER_INFO3 | 11 | +| SEVERITY_NUMBER_INFO4 | 12 | +| SEVERITY_NUMBER_WARN | 13 | +| SEVERITY_NUMBER_WARN2 | 14 | +| SEVERITY_NUMBER_WARN3 | 15 | +| SEVERITY_NUMBER_WARN4 | 16 | +| SEVERITY_NUMBER_ERROR | 17 | +| SEVERITY_NUMBER_ERROR2 | 18 | +| SEVERITY_NUMBER_ERROR3 | 19 | +| SEVERITY_NUMBER_ERROR4 | 20 | +| SEVERITY_NUMBER_FATAL | 21 | +| SEVERITY_NUMBER_FATAL2 | 22 | +| SEVERITY_NUMBER_FATAL3 | 23 | +| SEVERITY_NUMBER_FATAL4 | 24 | \ No newline at end of file diff --git a/pkg/ottl/contexts/ottlotelcol/README.md b/pkg/ottl/contexts/ottlotelcol/README.md index 8857bb97f515c..02eed0a896ca3 100644 --- a/pkg/ottl/contexts/ottlotelcol/README.md +++ b/pkg/ottl/contexts/ottlotelcol/README.md @@ -18,14 +18,13 @@ The following paths are supported. | otelcol.grpc.metadata | incoming gRPC metadata from the context | pcommon.Map | | otelcol.grpc.metadata[""] | values slice for a specific incoming gRPC metadata key | string or nil | - > [!NOTE] This context is read-only; any attempt to set these paths returns an error. ## Security and best practices -The `otelcol` context exposes client and request data that often contains **sensitive information**: authentication tokens (e.g. in `otelcol.client.auth.attributes`), HTTP headers and gRPC metadata (e.g. in `otelcol.client.metadata` and `otelcol.grpc.metadata`), and client address. -OTTL can **read** these values and **write** them into span attributes, log body, resource attributes, or metric labels. +The `otelcol` context exposes client and request data that often contains **sensitive information**: authentication tokens (e.g. in `otelcol.client.auth.attributes`), HTTP headers and gRPC metadata (e.g. in `otelcol.client.metadata` and `otelcol.grpc.metadata`), and client address. +OTTL can **read** these values and **write** them into span attributes, log body, resource attributes, or metric labels. Copying such data into telemetry can expose secrets to every system that receives it. ### Do not leak secrets into telemetry @@ -67,8 +66,8 @@ Treat the following (and similar) keys as sensitive. Do not copy their values in | `x-auth-token`, `x-access-token` | Access tokens | | `proxy-authorization` | Proxy credentials | -This applies to **`otelcol.client.metadata["key"]`** and **`otelcol.grpc.metadata["key"]`**. -For **`otelcol.client.auth.attributes`**, treat the **entire** map as sensitive unless you know the auth extension only exposes non-secret attributes. +This applies to **`otelcol.client.metadata["key"]`** and **`otelcol.grpc.metadata["key"]`**. +For **`otelcol.client.auth.attributes`**, treat the **entire** map as sensitive unless you know the auth extension only exposes non-secret attributes. HTTP header names are often normalized to lowercase; consider that when allowlisting or blocklisting. ### Recommended practices @@ -90,7 +89,6 @@ HTTP header names are often normalized to lowercase; consider that when allowlis | Allowlist and validate keys when copying to telemetry | Copy `otelcol.client.auth.attributes` or full metadata maps into exported data | | Document and review OTTL that uses `otelcol` context | Assume all client/metadata values are safe to store in telemetry | - ## Feature gate The `otelcol` context is temporally controlled by the **`ottl.contexts.enableOTelColContext`** [feature gate](https://github.com/open-telemetry/opentelemetry-collector/blob/main/featuregate/README.md#collector-feature-gates). diff --git a/pkg/ottl/contexts/ottlspan/README.md b/pkg/ottl/contexts/ottlspan/README.md index 9b464929584ec..a4af609dea475 100644 --- a/pkg/ottl/contexts/ottlspan/README.md +++ b/pkg/ottl/contexts/ottlspan/README.md @@ -65,8 +65,8 @@ The Span Context supports the enum names from the [traces proto](https://github. | SPAN_KIND_INTERNAL | 1 | | SPAN_KIND_SERVER | 2 | | SPAN_KIND_CLIENT | 3 | -| SPAN_KIND_PRODUCER | 4 | -| SPAN_KIND_CONSUMER | 5 | -| STATUS_CODE_UNSET | 0 | -| STATUS_CODE_OK | 1 | -| STATUS_CODE_ERROR | 2 | +| SPAN_KIND_PRODUCER | 4 | +| SPAN_KIND_CONSUMER | 5 | +| STATUS_CODE_UNSET | 0 | +| STATUS_CODE_OK | 1 | +| STATUS_CODE_ERROR | 2 | diff --git a/pkg/ottl/contexts/ottlspanevent/README.md b/pkg/ottl/contexts/ottlspanevent/README.md index b2e713656ddb3..1dda680be28e7 100644 --- a/pkg/ottl/contexts/ottlspanevent/README.md +++ b/pkg/ottl/contexts/ottlspanevent/README.md @@ -42,9 +42,9 @@ The Span Event Context supports the enum names from the [traces proto](https://g | SPAN_KIND_UNSPECIFIED | 0 | | SPAN_KIND_INTERNAL | 1 | | SPAN_KIND_SERVER | 2 | -| SPAN_KIND_CLIENT | 3 | -| SPAN_KIND_PRODUCER | 4 | -| SPAN_KIND_CONSUMER | 5 | -| STATUS_CODE_UNSET | 0 | -| STATUS_CODE_OK | 1 | -| STATUS_CODE_ERROR | 2 | +| SPAN_KIND_CLIENT | 3 | +| SPAN_KIND_PRODUCER | 4 | +| SPAN_KIND_CONSUMER | 5 | +| STATUS_CODE_UNSET | 0 | +| STATUS_CODE_OK | 1 | +| STATUS_CODE_ERROR | 2 | diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 9b8b19faa8e43..60fb30db60361 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -65,7 +65,7 @@ Available Editors: `append(target, Optional[value], Optional[values])` -The `append` function appends single or multiple string values to `target`. +The `append` function appends single or multiple string values to `target`. `append` converts scalar values into an array if the field exists but is not an array, and creates an array containing the provided values if the field doesn’t exist. Resulting field is always of type `pcommon.Slice` and will not convert the types of existing or new items in the slice. This means that it is possible to create a slice whose elements have different types. Be careful when using `append` to set attribute values, as this will produce values that are not possible to create through OpenTelemetry APIs [according to](https://opentelemetry.io/docs/specs/otel/common/#attribute) the OpenTelemetry specification. @@ -102,7 +102,6 @@ The key will be deleted from the map. Examples: - - `delete_key(log.attributes, "http.request.header.authorization")` - `delete_key(resource.attributes, "http.request.header.authorization")` @@ -119,7 +118,6 @@ All keys that match the pattern will be deleted from the map. Examples: - - `delete_matching_keys(log.attributes, "(?i).*password.*")` - `delete_matching_keys(resource.attributes, "(?i).*password.*")` @@ -136,7 +134,6 @@ All keys that match the pattern will remain in the map, while non matching keys Examples: - - `keep_matching_keys(log.attributes, "(?i).*version.*")` - `keep_matching_keys(resource.attributes, "(?i).*version.*")` @@ -145,11 +142,10 @@ Examples: `flatten(target, Optional[prefix], Optional[depth], Optional[resolveConflicts])` -The `flatten` function flattens a `pcommon.Map` by moving items from nested maps to the root. +The `flatten` function flattens a `pcommon.Map` by moving items from nested maps to the root. `target` is a path expression to a `pcommon.Map` type field. `prefix` is an optional string. `depth` is an optional non-negative int, `resolveConflicts` resolves the potential conflicts in the map keys by adding a number suffix starting with `0` from the first duplicated key. - For example, the following map ```json @@ -163,7 +159,7 @@ For example, the following map } ``` -is converted to +is converted to ```json { @@ -261,16 +257,12 @@ Examples: - `flatten(resource.attributes)` - - `flatten(metric.cache, "k8s", 4)` - - `flatten(log.body, depth=2)` - - `flatten(body, resolveConflicts=true)` - ### keep_keys `keep_keys(target, keys[])` @@ -285,7 +277,6 @@ Examples: - `keep_keys(log.attributes, ["http.method"])` - - `keep_keys(resource.attributes, ["http.method", "http.route", "http.url"])` ### limit @@ -308,7 +299,6 @@ Examples: - `limit(log.attributes, 100, [])` - - `limit(resource.attributes, 50, ["http.host", "http.method"])` ### merge_maps @@ -331,10 +321,8 @@ Examples: - `merge_maps(log.attributes, ParseJSON(log.body), "upsert")` - - `merge_maps(log.attributes, ParseJSON(log.attributes["kubernetes"]), "update")` - - `merge_maps(log.attributes, resource.attributes, "insert")` ### replace_all_matches @@ -437,13 +425,10 @@ Examples: - `set(resource.attributes["http.path"], "/foo")` - - `set(metric.name, resource.attributes["http.route"])` - - `set(span.trace_state["svc"], "example")` - - `set(span.attributes["source"], span.trace_state["source"])` ### truncate_all @@ -580,7 +565,6 @@ Examples: - `Base64Decode("aGVsbG8gd29ybGQ=")` - - `Base64Decode(resource.attributes["encoded field"])` ### Base64Encode @@ -596,13 +580,10 @@ Examples: - `Base64Encode("test string")` - - `Base64Encode(resource.attributes["field"])` - - `Base64Encode(body, "base64-url")` - - `Base64Encode(attributes["data"], "base64-raw")` ### Bool @@ -629,10 +610,8 @@ Examples: - `Bool(log.attributes["truthy_attribute"])` - - `Bool("true")` - - `Bool("0")` ### Decode @@ -648,7 +627,6 @@ Examples: - `Decode("aGVsbG8gd29ybGQ=", "base64")` - - `Decode(resource.attributes["encoded field"], "us-ascii")` ### Coalesce @@ -665,7 +643,6 @@ Examples: - `Coalesce([attributes["user.id"], attributes["enduser.id"], "unknown"])` - - `Coalesce([resource.attributes["deployment.environment.name"], resource.attributes["deployment.environment"]])` ### CommunityID @@ -686,10 +663,8 @@ Examples: - `CommunityID(attributes["source.ip"], attributes["source.port"], attributes["destination.ip"], attributes["destination.port"], "TCP", 1)` - - `CommunityID("192.168.1.1", 54321, "10.0.0.1", 90, "UDP", 2)` - ### Concat `Concat(values[], delimiter)` @@ -704,10 +679,8 @@ Examples: - `Concat([span.attributes["http.method"], span.attributes["http.path"]], ": ")` - - `Concat([metric.name, 1], " ")` - - `Concat(["HTTP method is: ", span.attributes["http.method"]], "")` ### ContainsValue @@ -840,7 +813,6 @@ Examples: - `Double(log.attributes["http.status_code"])` - - `Double("2.0")` ### Duration @@ -879,14 +851,14 @@ Examples: `ExtractGrokPatterns(target, pattern, Optional[namedCapturesOnly], Optional[patternDefinitions])` -The `ExtractGrokPatterns` Converter parses unstructured data into a format that is structured and queryable. +The `ExtractGrokPatterns` Converter parses unstructured data into a format that is structured and queryable. It returns a `pcommon.Map` struct that is a result of extracting named capture groups from the target string. If no matches are found then an empty `pcommon.Map` is returned. -- `target` is a Getter that returns a string. -- `pattern` is a grok pattern string. -- `namedCapturesOnly` (optional) specifies if non-named captures should be returned. -- `patternDefinitions` (optional) is a list of custom pattern definition strings used inside `pattern` in the form of `PATTERN_NAME=PATTERN`. -This parameter lets you define your own custom patterns to improve readability when the extracted `pattern` is not part of the default set or when you need custom naming. +- `target` is a Getter that returns a string. +- `pattern` is a grok pattern string. +- `namedCapturesOnly` (optional) specifies if non-named captures should be returned. +- `patternDefinitions` (optional) is a list of custom pattern definition strings used inside `pattern` in the form of `PATTERN_NAME=PATTERN`. +This parameter lets you define your own custom patterns to improve readability when the extracted `pattern` is not part of the default set or when you need custom naming. If `target` is not a string or nil `ExtractGrokPatterns` returns an error. If `pattern` does not contain at least 1 named capture group and `namedCapturesOnly` is set to `true` then `ExtractPatterns` errors on startup. @@ -905,7 +877,6 @@ Supported types are `int`, `long`, `double`, `float` and boolean The [Elastic Go-Grok](https://github.com/elastic/go-grok) ships with numerous predefined grok patterns that simplify working with grok. In collector Complete set is included consisting of a default set and all additional sets adding product/tool specific capabilities (like [aws](https://github.com/elastic/go-grok/blob/main/patterns/aws.go) or [java](https://github.com/elastic/go-grok/blob/main/patterns/java.go) patterns). - Default set consists of: | Name | Example | @@ -959,13 +930,12 @@ Examples: Note that `USERNAME` is in the default pattern set and does not need to be redefined. - - Target: `smith:pass123:1001:1000:J Smith,1234,(234)567-8910,(234)567-1098,email:/home/smith:/bin/sh` + - Target: `smith:pass123:1001:1000:J Smith,1234,(234)567-8910,(234)567-1098,email:/home/smith:/bin/sh` - - Return values: + - Return values: - `user.name`: smith - `user.password`: pass123 - ### FNV `FNV(value)` @@ -982,7 +952,6 @@ Examples: - `FNV(resource.attributes["device.name"])` - - `FNV("name")` ### Format @@ -1119,7 +1088,6 @@ Examples: - `HasPrefix(resource.attributes["service.name"], "ingest_")` - - `HasPrefix("ingest_service", "ingest_")` ### HasSuffix @@ -1138,7 +1106,6 @@ Examples: - `HasSuffix(resource.attributes["service.name"], "_service")` - - `HasSuffix("ingest_service", "_service")` ### Hex @@ -1165,7 +1132,6 @@ Examples: - `Hex(span.attributes["http.status_code"])` - - `Hex(2.0)` ### Hour @@ -1266,7 +1232,6 @@ Examples: - `Int(log.attributes["http.status_code"])` - - `Int("2.0")` ### IsBool @@ -1286,13 +1251,10 @@ Examples: - `IsBool(false)` - - `IsBool(pcommon.NewValueBool(false))` - - `IsBool(42)` - - `IsBool(resource.attributes["any key"])` ### IsDouble @@ -1372,7 +1334,6 @@ Examples: - `IsMap(log.body)` - - `IsMap(log.attributes["maybe a map"])` ### IsMatch @@ -1398,7 +1359,6 @@ Examples: - `IsMatch(span.attributes["http.path"], "foo")` - - `IsMatch("string", ".*ring")` ### IsList @@ -1444,7 +1404,7 @@ The `Keys` Converter returns a slice containing all the keys from the given map. The returned type is `pcommon.Slice`. Examples: -- + - `Keys(resource.attributes)` - `Keys({"k1":"v1", "k2": "v2"})` @@ -1485,7 +1445,6 @@ Examples: - `Log(span.attributes["duration_ms"])` - - `Int(Log(span.attributes["duration_ms"])` ### IsValidLuhn @@ -1685,13 +1644,10 @@ Examples: - `ParseCSV("999-999-9999,Joe Smith,joe.smith@example.com", "phone,name,email")` - - `ParseCSV(log.body, "phone|name|email", delimiter="|")` - - `ParseCSV(log.attributes["csv_line"], log.attributes["csv_headers"], delimiter="|", headerDelimiter=",", mode="lazyQuotes")` - - `ParseCSV("\"555-555-5556,Joe Smith\",joe.smith@example.com", "phone,name,email", mode="ignoreQuotes")` ### ParseInt @@ -1706,7 +1662,7 @@ The `ParseInt` Converter interprets a string `target` in the given `base` (0, 2 If the `base` argument is 0, the true base is implied by the string's prefix following the sign (if present): 2 for "0b", 8 for "0" or "0o", 16 for "0x", and 10 otherwise. When the `base` value is 0, underscore characters are permitted as defined by the Go syntax for [integer literals](https://go.dev/ref/spec#Integer_literals). -Examples of `ParseInt` behavior when `base` is 0: +Examples of `ParseInt` behavior when `base` is 0: - `ParseInt("0b1111_0000", 0) -> 240` - `ParseInt("0b10110", 0) -> 22` - `ParseInt("0xFF", 0) -> 255` @@ -1749,13 +1705,10 @@ Examples: - `ParseJSON("{\"attr\":true}")` - - `ParseJSON("[\"attr1\",\"attr2\"]")` - - `ParseJSON(resource.attributes["kubernetes"])` - - `ParseJSON(log.body)` ### ParseKeyValue @@ -1786,7 +1739,7 @@ The `ParseSeverity` converter returns a `string` that represents one of the log `target` is a Getter that returns a string or an integer. `severityMapping` is a map containing the log levels, and a list of values they are mapped from. These values can be either strings, or map items containing a numeric range, defined by a `min` and `max` key (inclusive bounds), for the given log level. -A value will be mapped to the given log level if any of these conditions are true. +A value will be mapped to the given log level if any of these conditions are true. For example, the following mapping will map to the `info` level, if the `target` is either a string with the value `inf`, or an integer in the range `[200,299]`: @@ -2129,7 +2082,6 @@ Examples: - `SHA1(resource.attributes["device.name"])` - - `SHA1("name")` **Note:** [According to the National Institute of Standards and Technology (NIST)](https://csrc.nist.gov/projects/hash-functions), SHA1 is no longer a recommended hash function. It should be avoided except when required for compatibility. New uses should prefer a SHA-2 family function (such as SHA-256 or SHA-512) whenever possible. @@ -2177,14 +2129,14 @@ The `SliceToMap` converter converts a slice of objects to a map. The arguments a - `target`: A list of maps containing the entries to be converted. - `keyPath`: An optional string array that determines the name of the keys for the map entries by pointing to the value of an attribute within each slice item. Note that -if `keyPath` is provided, it must resolve to a string value, otherwise the converter will not be able to convert the item to a map entry. If `keyPath` isn't provided, the string representation of the index when looping through objects in the slice will be the key for the object in the output map. +if `keyPath` is provided, it must resolve to a string value, otherwise the converter will not be able to convert the item to a map entry. If `keyPath` isn't provided, the string representation of the index when looping through objects in the slice will be the key for the object in the output map. - `valuePath`: This optional string array determines which attribute should be used as the value for the map entry. If no `valuePath` is defined, the value of the map entry will be the same as the original slice item. Examples: -The examples below will convert the following input: +The examples below will convert the following input: ```yaml attributes: @@ -2252,11 +2204,11 @@ Once the `SliceToMap` function has been applied to a value, the converted entrie The `Sort` Converter sorts the `target` array in either ascending or descending order. -`target` is an array or `pcommon.Slice` typed field containing the elements to be sorted. +`target` is an array or `pcommon.Slice` typed field containing the elements to be sorted. `order` is a string specifying the sort order. Must be either `asc` or `desc`. The default value is `asc`. -The Sort Converter preserves the data type of the original elements while sorting. +The Sort Converter preserves the data type of the original elements while sorting. The behavior varies based on the types of elements in the target slice: | Element Types | Sorting Behavior | Return Value | @@ -2330,7 +2282,6 @@ Examples: - `set(resource.attributes["service.name"], TrimPrefix(resource.attributes["service.name"], "ingest_"))` - - `TrimPrefix("ingest_service", "ingest_")` ### TrimSuffix @@ -2349,7 +2300,6 @@ Examples: - `set(resource.attributes["service.name"], TrimSuffix(resource.attributes["service.name"], "_service"))` - - `TrimSuffix("ingest_service", "_service")` ### String @@ -2451,7 +2401,7 @@ When loading `location`, this function will look for the IANA Time Zone database - a directory or uncompressed zip file named by the ZONEINFO environment variable - on a Unix system, the system standard installation location - $GOROOT/lib/time/zoneinfo.zip -- the `time/tzdata` package, if it was imported. +- the `time/tzdata` package, if it was imported. When building a Collector binary, importing `time/tzdata` in any Go source file will bundle the database into the binary, which guarantees the lookups will work regardless of the setup on the host setup. Note this will add roughly 500kB to binary size. @@ -2464,7 +2414,7 @@ Examples: - `Time("2012-11-01T22:08:41+0000 EST", "%Y-%m-%dT%H:%M:%S%z %Z")` - `Time("2023-05-26 12:34:56", "%Y-%m-%d %H:%M:%S", "America/New_York")` -`locale` specifies the input language of the `target` value. It is used to interpret timestamp values written in a specific language, +`locale` specifies the input language of the `target` value. It is used to interpret timestamp values written in a specific language, ensuring that the function can correctly parse the localized month names, day names, and periods of the day based on the provided language. The value must be a well-formed BCP 47 language tag, and a known [CLDR](https://cldr.unicode.org) v45 locale. @@ -2493,10 +2443,10 @@ Examples: The `ToKeyValueString` Converter takes a `pcommon.Map` and converts it to a `string` of key value pairs. -- `target` is a Getter that returns a `pcommon.Map`. -- `delimiter` is an optional string that is used to join keys and values, the default is `=`. +- `target` is a Getter that returns a `pcommon.Map`. +- `delimiter` is an optional string that is used to join keys and values, the default is `=`. - `pair_delimiter` is an optional string that is used to join key value pairs, the default is a single space (` `). -- `sort_output` is an optional bool that is used to deterministically sort the keys of the output string. It should only be used if the output is required to be in the same order each time, as it introduces some performance overhead. +- `sort_output` is an optional bool that is used to deterministically sort the keys of the output string. It should only be used if the output is required to be in the same order each time, as it introduces some performance overhead. For example, the following map `{"k1":"v1","k2":"v2","k3":"v3"}` will use default delimiters and be converted into the following string: @@ -2504,7 +2454,7 @@ For example, the following map `{"k1":"v1","k2":"v2","k3":"v3"}` will use defaul `k1=v1 k2=v2 k3=v3` ``` -**Note:** Any nested arrays or maps will be represented as a JSON string. It is recommended to [flatten](#flatten) `target` before using this function. +**Note:** Any nested arrays or maps will be represented as a JSON string. It is recommended to [flatten](#flatten) `target` before using this function. For example, `{"k1":"v1","k2":{"k3":"v3","k4":["v4","v5"]}}` will be converted to: @@ -2575,7 +2525,6 @@ Examples: - `TraceID(0x00000000000000000000000000000000)` - `TraceID("a389023abaa839283293ed323892389d")` - ### TruncateTime `TruncateTime(time, duration)` @@ -2671,7 +2620,6 @@ The `UserAgent` Converter parses the string argument trying to match it against The results of the parsing are returned as a map containing `user_agent.name`, `user_agent.version`, `user_agent.original`, `os.name`, and `os.version` as defined in semconv v1.34.0. `os.name` and `os.version` are omitted if empty. - Parsing is done using the [uap-go package](https://github.com/ua-parser/uap-go). The specific formats it recognizes can be found [here](https://github.com/ua-parser/uap-core/blob/master/regexes.yaml). Examples: @@ -2712,7 +2660,7 @@ This URL object includes properties for the URL’s domain, path, fragment, port - `URL("http://www.example.com")` -results in +results in ``` "url.original": "http://www.example.com", "url.scheme": "http", @@ -2722,7 +2670,7 @@ results in - `URL("http://myusername:mypassword@www.example.com:80/foo.gif?key1=val1&key2=val2#fragment")` -results in +results in ``` "url.path": "/foo.gif", "url.fragment": "fragment", diff --git a/pkg/pdatatest/README.md b/pkg/pdatatest/README.md index 05a5c4337ef83..555535c115343 100644 --- a/pkg/pdatatest/README.md +++ b/pkg/pdatatest/README.md @@ -1,65 +1,65 @@ # pdatatest -This module provides a test helpers for comparing metric, log and traces. The main functions are: -- `pmetrictest.CompareMetrics` -- `plogtest.CompareLogs` -- `ptracetest.CompareTraces` +This module provides a test helpers for comparing metric, log and traces. The main functions are: +- `pmetrictest.CompareMetrics` +- `plogtest.CompareLogs` +- `ptracetest.CompareTraces` -These functions compare the actual result with the expected result and return an error if they are not equal. -The error contains a detailed description of the differences. The module also provides several options to customize -the comparison by ignoring certain fields, attributes, or slices order. The module also provides helper functions +These functions compare the actual result with the expected result and return an error if they are not equal. +The error contains a detailed description of the differences. The module also provides several options to customize +the comparison by ignoring certain fields, attributes, or slices order. The module also provides helper functions for comparing other embedded pdata types such as `pmetric.ResourceMetrics`, `pmetric.ScopeMetrics`, `plog.LogRecord`, -`ptrace.Span`, etc. +`ptrace.Span`, etc. ## Typical Usage ```go func TestMetricsScraper(t *testing.T) { - scraper := newScraper(componenttest.NewNopReceiverCreateSettings(), createDefaultConfig().(*Config)) - require.NoError(t, scraper.start(context.Background(), componenttest.NewNopHost())) - actualMetrics, err := require.NoError(t, scraper.scrape(context.Background())) - require.NoError(t, err) + scraper := newScraper(componenttest.NewNopReceiverCreateSettings(), createDefaultConfig().(*Config)) + require.NoError(t, scraper.start(context.Background(), componenttest.NewNopHost())) + actualMetrics, err := require.NoError(t, scraper.scrape(context.Background())) + require.NoError(t, err) - expectedFile, err := readMetrics(filepath.Join("testdata", "expected.json")) - require.NoError(err) + expectedFile, err := readMetrics(filepath.Join("testdata", "expected.json")) + require.NoError(err) - require.NoError(t, pmetrictest.CompareMetrics(expectedMetrics, actualMetrics, pmetrictest.IgnoreStartTimestamp(), - pmetrictest.IgnoreTimestamp())) + require.NoError(t, pmetrictest.CompareMetrics(expectedMetrics, actualMetrics, pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreTimestamp())) } ``` ```go func TestLogsReceiver(t *testing.T) { - sink := &consumertest.LogsSink{} - rcvr := newLogsReceiver(createDefaultConfig().(*Config), zap.NewNop(), sink) - rcvr.client = defaultMockClient() - require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost())) - require.Eventually(t, func() bool { - return sink.LogRecordCount() > 0 - }, 2*time.Second, 10*time.Millisecond) - err = rcvr.Shutdown(context.Background()) - require.NoError(t, err) - actualLogs := sink.AllLogs()[0] + sink := &consumertest.LogsSink{} + rcvr := newLogsReceiver(createDefaultConfig().(*Config), zap.NewNop(), sink) + rcvr.client = defaultMockClient() + require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost())) + require.Eventually(t, func() bool { + return sink.LogRecordCount() > 0 + }, 2*time.Second, 10*time.Millisecond) + err = rcvr.Shutdown(context.Background()) + require.NoError(t, err) + actualLogs := sink.AllLogs()[0] - expectedLogs, err := readLogs(filepath.Join("testdata", "logs", "expected.json")) - require.NoError(t, err) + expectedLogs, err := readLogs(filepath.Join("testdata", "logs", "expected.json")) + require.NoError(t, err) - require.NoError(t, pmetrictest.CompareLogs(expectedLogs, actualLogs)) + require.NoError(t, pmetrictest.CompareLogs(expectedLogs, actualLogs)) } ``` ```go func TestTraceProcessor(t *testing.T) { - nextTrace := new(consumertest.TracesSink) - tp, err := newTracesProcessor(NewFactory().CreateDefaultConfig(), nextTrace) - traces := generateTraces() - tp.ConsumeTraces(ctx, traces) - actualTraces := nextTrace.AllTraces()[0] + nextTrace := new(consumertest.TracesSink) + tp, err := newTracesProcessor(NewFactory().CreateDefaultConfig(), nextTrace) + traces := generateTraces() + tp.ConsumeTraces(ctx, traces) + actualTraces := nextTrace.AllTraces()[0] - expectedTraces, err := readTraces(filepath.Join("testdata", "traces", "expected.json")) - require.NoError(t, err) - - require.NoError(t, ptracetest.CompareTraces(expectedTraces, actualTraces, ptracetest.IgnoreStartTimestamp(), - ptracetest.IgnoreEndTimestamp())) + expectedTraces, err := readTraces(filepath.Join("testdata", "traces", "expected.json")) + require.NoError(t, err) + + require.NoError(t, ptracetest.CompareTraces(expectedTraces, actualTraces, ptracetest.IgnoreStartTimestamp(), + ptracetest.IgnoreEndTimestamp())) } ``` \ No newline at end of file diff --git a/pkg/stanza/docs/operators/README.md b/pkg/stanza/docs/operators/README.md index 992f8d3f20896..8618bbc8d0d78 100644 --- a/pkg/stanza/docs/operators/README.md +++ b/pkg/stanza/docs/operators/README.md @@ -3,7 +3,6 @@ An operator is the most basic unit of log processing. Each operator fulfills a s For instance, a user may read lines from a file using the `file_input` operator. From there, the results of this operation may be sent to a `regex_parser` operator that creates fields based on a regex pattern. And then finally, these results may be sent to a `file_output` operator that writes each line to a file on disk. - ## What operators are available? Inputs: diff --git a/pkg/stanza/docs/operators/add.md b/pkg/stanza/docs/operators/add.md index f9da77315119d..3b2ea21105603 100644 --- a/pkg/stanza/docs/operators/add.md +++ b/pkg/stanza/docs/operators/add.md @@ -13,7 +13,6 @@ The `add` operator adds a value to an `entry`'s `body`, `attributes`, or `resour | `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](../types/on_error.md). | | `if` | | An [expression](../types/expression.md) that, when set, will be evaluated to determine whether this operator should be used for the given entry. This allows you to do easy conditional parsing without branching logic with routers. | - ### Example Configurations:
diff --git a/pkg/stanza/docs/operators/container.md b/pkg/stanza/docs/operators/container.md index bad7ede0639ca..a4a08f927ba4b 100644 --- a/pkg/stanza/docs/operators/container.md +++ b/pkg/stanza/docs/operators/container.md @@ -17,7 +17,6 @@ The `container` operator parses logs in `docker`, `cri-o` and `containerd` forma | `if` | | An [expression](../types/expression.md) that, when set, will be evaluated to determine whether this operator should be used for the given entry. This allows you to do easy conditional parsing without branching logic with routers. | | `severity` | `nil` | An optional [severity](../types/severity.md) block which will parse a severity field before passing the entry to the output operator. | - ### Embedded Operations The `container` parser can be configured to embed certain operations such as the severity parsing. For more information, see [complex parsers](../types/parsers.md#complex-parsers). @@ -243,10 +242,9 @@ Configuration: - #### Parse multiline logs and recombine into a single one -If you are using the Docker format (or log tag indicators are not working), +If you are using the Docker format (or log tag indicators are not working), you can recombine multiline logs into a single one. Configuration: diff --git a/pkg/stanza/docs/operators/file_input.md b/pkg/stanza/docs/operators/file_input.md index af306d5eaba63..dbf7dd2a48c5e 100644 --- a/pkg/stanza/docs/operators/file_input.md +++ b/pkg/stanza/docs/operators/file_input.md @@ -66,7 +66,7 @@ To avoid the data loss, choose move/create rotation method and set `max_concurre ### Supported encodings -| Key | Description +| Key | Description | | --- | --- | | `nop` | No encoding validation. Treats the file as a stream of raw bytes | | `utf-8` | UTF-8 encoding | diff --git a/pkg/stanza/docs/operators/file_output.md b/pkg/stanza/docs/operators/file_output.md index a5b1fca4e5caa..7428bda92c0a2 100644 --- a/pkg/stanza/docs/operators/file_output.md +++ b/pkg/stanza/docs/operators/file_output.md @@ -10,7 +10,6 @@ The `file_output` operator will write log entries to a file. By default, they wi | `path` | required | The file path to which entries will be written. | | `format` | | A [go template](https://golang.org/pkg/text/template/) that will be used to render each entry into a log line. | - ### Example Configurations #### Simple configuration diff --git a/pkg/stanza/docs/operators/json_array_parser.md b/pkg/stanza/docs/operators/json_array_parser.md index 734fed54b5601..fd2fefe0ebb5b 100644 --- a/pkg/stanza/docs/operators/json_array_parser.md +++ b/pkg/stanza/docs/operators/json_array_parser.md @@ -33,14 +33,13 @@ json array after parsing: ["Hello", 42, {"name": "Alice", "age": 25}, [1, 2, 3], true, null] ``` -Notice that for this example, the current parser will parse every nested object as a string and so the result is actually this - +Notice that for this example, the current parser will parse every nested object as a string and so the result is actually this - ```json ["Hello", 42, "{\"name\": \"Alice\", \"age\": 25}", "[1, 2, 3]", true, null] ``` More information on json arrays can be found [here](https://json-schema.org/understanding-json-schema/reference/array) - ### Configuration Fields | Field | Default | Description | @@ -60,7 +59,7 @@ The `json_array_parser` can be configured to embed certain operations such as ti ### Example Configurations -#### Parse the field `body` with a json array parser into an attributes field +#### Parse the field `body` with a json array parser into an attributes field Configuration: diff --git a/pkg/stanza/docs/operators/json_parser.md b/pkg/stanza/docs/operators/json_parser.md index 2f8f2c2286bdc..8d4f5a8c8a58d 100644 --- a/pkg/stanza/docs/operators/json_parser.md +++ b/pkg/stanza/docs/operators/json_parser.md @@ -20,10 +20,8 @@ The `json_parser` operator parses the string-type field selected by `parse_from` The `json_parser` can be configured to embed certain operations such as timestamp and severity parsing. For more information, see [complex parsers](../types/parsers.md#complex-parsers). - ### Example Configurations - #### Parse the field `message` as JSON Configuration: diff --git a/pkg/stanza/docs/operators/move.md b/pkg/stanza/docs/operators/move.md index 03b88f7a88a17..ffcc940c89fed 100644 --- a/pkg/stanza/docs/operators/move.md +++ b/pkg/stanza/docs/operators/move.md @@ -275,4 +275,3 @@ Merge a layer to the body - diff --git a/pkg/stanza/docs/operators/noop.md b/pkg/stanza/docs/operators/noop.md index 7979cbb0a4847..8493018c51648 100644 --- a/pkg/stanza/docs/operators/noop.md +++ b/pkg/stanza/docs/operators/noop.md @@ -9,7 +9,6 @@ The `noop` operator makes no changes to a entry. It is sometimes useful as a ter | `id` | `noop` | A unique identifier for the operator. | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries. | - ### Example Configuration: Process logs according to some criteria, then direct all logs to a `noop` operator before emitting from the receiver. diff --git a/pkg/stanza/docs/operators/regex_parser.md b/pkg/stanza/docs/operators/regex_parser.md index 9b29401209ff5..27795fe9bad96 100644 --- a/pkg/stanza/docs/operators/regex_parser.md +++ b/pkg/stanza/docs/operators/regex_parser.md @@ -36,7 +36,6 @@ Setting the size to 0 will disable the cache. This is the default. ### Example Configurations - #### Parse the field `message` with a regular expression Configuration: diff --git a/pkg/stanza/docs/operators/regex_replace.md b/pkg/stanza/docs/operators/regex_replace.md index 644e1885b9161..fcf877d146e0b 100644 --- a/pkg/stanza/docs/operators/regex_replace.md +++ b/pkg/stanza/docs/operators/regex_replace.md @@ -28,7 +28,7 @@ This operator makes use of [Go regular expression](https://github.com/google/re2 ### Example Configurations -#### Collapse spaces +#### Collapse spaces Configuration: ```yaml diff --git a/pkg/stanza/docs/operators/router.md b/pkg/stanza/docs/operators/router.md index 11556b638ac2c..335073e382fcd 100644 --- a/pkg/stanza/docs/operators/router.md +++ b/pkg/stanza/docs/operators/router.md @@ -24,7 +24,6 @@ An entry that does not match any of the routes is dropped and not processed furt | `expr` | required | An [expression](../types/expression.md) that returns a boolean. The body of the routed entry is available as `$`. | | `attributes` | {} | A map of `key: value` pairs to add to an entry that matches the route. | - ### Examples #### Forward entries to different parsers based on content diff --git a/pkg/stanza/docs/operators/scope_name_parser.md b/pkg/stanza/docs/operators/scope_name_parser.md index 839a8deb42e81..6c662796f8614 100644 --- a/pkg/stanza/docs/operators/scope_name_parser.md +++ b/pkg/stanza/docs/operators/scope_name_parser.md @@ -11,7 +11,6 @@ The `scope_name_parser` operator sets the scope name on an entry by parsing a va | `parse_from` | `body` | A [field](../types/field.md) that indicates the field to be parsed as the scope name. | | `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](../types/on_error.md). | - ### Example Configurations Several detailed examples are available [here](../types/scope_name.md). diff --git a/pkg/stanza/docs/operators/stdout.md b/pkg/stanza/docs/operators/stdout.md index cbd2d66860ed9..ddbbd55245aed 100644 --- a/pkg/stanza/docs/operators/stdout.md +++ b/pkg/stanza/docs/operators/stdout.md @@ -9,7 +9,6 @@ or running one-time batch processing jobs. | --- | --- | --- | | `id` | `stdout` | A unique identifier for the operator. | - ### Example Configurations #### Simple configuration diff --git a/pkg/stanza/docs/operators/syslog_input.md b/pkg/stanza/docs/operators/syslog_input.md index 3ed7a9707e320..9d2c19b1bf621 100644 --- a/pkg/stanza/docs/operators/syslog_input.md +++ b/pkg/stanza/docs/operators/syslog_input.md @@ -15,9 +15,6 @@ The `syslog_input` operator listens for syslog format logs from UDP/TCP packages | `resource` | {} | A map of `key: value` pairs to add to the entry's resource. | | `on_error` | `send` | The behavior of the syslog parser if it encounters an error. See [on_error](../types/on_error.md). | - - - ### Example Configurations #### Simple @@ -41,4 +38,3 @@ UDP Configuration: protocol: rfc3164 location: UTC ``` - diff --git a/pkg/stanza/docs/operators/syslog_parser.md b/pkg/stanza/docs/operators/syslog_parser.md index 4196f0772297c..2b59100cd9f6c 100644 --- a/pkg/stanza/docs/operators/syslog_parser.md +++ b/pkg/stanza/docs/operators/syslog_parser.md @@ -26,7 +26,6 @@ The `syslog_parser` can be configured to embed certain operations such as timest ### Example Configurations - #### Parse the field `message` as syslog Configuration: diff --git a/pkg/stanza/docs/operators/tcp_input.md b/pkg/stanza/docs/operators/tcp_input.md index 3fa92c2db6a8d..ebed137bca4e1 100644 --- a/pkg/stanza/docs/operators/tcp_input.md +++ b/pkg/stanza/docs/operators/tcp_input.md @@ -14,7 +14,7 @@ The `tcp_input` operator listens for logs on one or more TCP connections. The op | `attributes` | {} | A map of `key: value` pairs to add to the entry's attributes. | | `one_log_per_packet` | false | Skip log tokenization, set to true if logs contains one log per record and multiline is not used. This will improve performance. | | `resource` | {} | A map of `key: value` pairs to add to the entry's resource. | -| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention][https://github.com/open-telemetry/semantic-conventions/blob/main/docs/registry/attributes/network.md#network-attributes]. | +| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/registry/attributes/network.md#network-attributes). | | `multiline` | | A `multiline` configuration block. See below for details. | | `preserve_leading_whitespaces` | false | Whether to preserve leading whitespaces. | | `preserve_trailing_whitespaces` | false | Whether to preserve trailing whitespaces. | @@ -43,7 +43,7 @@ The `omit_pattern` setting can be used to omit the start/end pattern from each e #### Supported encodings -| Key | Description +| Key | Description | | --- | --- | | `nop` | No encoding validation. Treats the file as a stream of raw bytes | | `utf-8` | UTF-8 encoding | diff --git a/pkg/stanza/docs/operators/time_parser.md b/pkg/stanza/docs/operators/time_parser.md index ab22627b7be73..b25b3e41952bb 100644 --- a/pkg/stanza/docs/operators/time_parser.md +++ b/pkg/stanza/docs/operators/time_parser.md @@ -16,7 +16,6 @@ The `time_parser` operator sets the timestamp on an entry by parsing a value fro | `if` | | An [expression](../types/expression.md) that, when set, will be evaluated to determine whether this operator should be used for the given entry. This allows you to do easy conditional parsing without branching logic with routers. | | `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](../types/on_error.md). | - ### Example Configurations Several detailed examples are available [here](../types/timestamp.md). diff --git a/pkg/stanza/docs/operators/trace_parser.md b/pkg/stanza/docs/operators/trace_parser.md index b7f74640ae26f..0e2d7540c73f5 100644 --- a/pkg/stanza/docs/operators/trace_parser.md +++ b/pkg/stanza/docs/operators/trace_parser.md @@ -13,7 +13,6 @@ The `trace_parser` operator sets the trace on an entry by parsing a value from t | `trace_flags.parse_from` | `trace_flags` | A [field](../types/field.md) that indicates the field to be parsed as trace flags. | | `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](../types/on_error.md). | - ### Example Configurations Several detailed examples are available [here](../types/trace.md). diff --git a/pkg/stanza/docs/operators/udp_input.md b/pkg/stanza/docs/operators/udp_input.md index 5f3bd43dac5ef..9152b266f2834 100644 --- a/pkg/stanza/docs/operators/udp_input.md +++ b/pkg/stanza/docs/operators/udp_input.md @@ -12,7 +12,7 @@ The `udp_input` operator listens for logs from UDP packets. | `attributes` | {} | A map of `key: value` pairs to add to the entry's attributes. | | `one_log_per_packet` | false | Skip log tokenization, set to true if logs contains one log per record and multiline is not used. This will improve performance. | | `resource` | {} | A map of `key: value` pairs to add to the entry's resource. | -| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention][https://github.com/open-telemetry/semantic-conventions/blob/cee22ec91448808ebcfa53df689c800c7171c9e1/docs/general/attributes.md#other-network-attributes]. | +| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention](https://github.com/open-telemetry/semantic-conventions/blob/cee22ec91448808ebcfa53df689c800c7171c9e1/docs/general/attributes.md#other-network-attributes). | | `multiline` | | A `multiline` configuration block. See below for details. | | `preserve_leading_whitespaces` | false | Whether to preserve leading whitespaces. | | `preserve_trailing_whitespaces` | false | Whether to preserve trailing whitespaces. | @@ -33,7 +33,7 @@ The `omit_pattern` setting can be used to omit the start/end pattern from each e #### Supported encodings -| Key | Description +| Key | Description | | --- | --- | | `nop` | No encoding validation. Treats the file as a stream of raw bytes | | `utf-8` | UTF-8 encoding | diff --git a/pkg/stanza/docs/operators/uri_parser.md b/pkg/stanza/docs/operators/uri_parser.md index 0b5a7fd125cc3..6fa891b2d923d 100644 --- a/pkg/stanza/docs/operators/uri_parser.md +++ b/pkg/stanza/docs/operators/uri_parser.md @@ -39,10 +39,8 @@ The following fields are returned. Empty fields are not returned. | path | `string` | `"/v1/app"` | URI request [path](https://tools.ietf.org/html/rfc3986#section-3.3). | | query | `map[string][]string` | `"query":{"user":["admin"]}` | Parsed URI [query string](https://tools.ietf.org/html/rfc3986#section-3.4). | - ### Example Configurations - #### Parse the field `body.message` as absolute URI Configuration: diff --git a/pkg/stanza/docs/operators/windows_eventlog_input.md b/pkg/stanza/docs/operators/windows_eventlog_input.md index 4c388b8eb8a7f..3cc3f1dee721f 100644 --- a/pkg/stanza/docs/operators/windows_eventlog_input.md +++ b/pkg/stanza/docs/operators/windows_eventlog_input.md @@ -34,24 +34,24 @@ Output entry sample: "timestamp": "2020-04-30T12:10:17.656726-04:00", "severity": 30, "body": { - "event_id": { - "qualifiers": 0, - "id": 1000 - }, - "provider": { - "name": "provider name", - "guid": "provider guid", - "event_source": "event source" - }, - "system_time": "2020-04-30T12:10:17.656726789Z", - "computer": "example computer", - "channel": "application", - "record_id": 1, - "level": "Information", - "message": "example message", - "task": "example task", - "opcode": "example opcode", - "keywords": ["example keyword"] - } + "event_id": { + "qualifiers": 0, + "id": 1000 + }, + "provider": { + "name": "provider name", + "guid": "provider guid", + "event_source": "event source" + }, + "system_time": "2020-04-30T12:10:17.656726789Z", + "computer": "example computer", + "channel": "application", + "record_id": 1, + "level": "Information", + "message": "example message", + "task": "example task", + "opcode": "example opcode", + "keywords": ["example keyword"] + } } ``` diff --git a/pkg/stanza/docs/types/bytesize.md b/pkg/stanza/docs/types/bytesize.md index fe1c0fac3e88c..94c27575a57ec 100644 --- a/pkg/stanza/docs/types/bytesize.md +++ b/pkg/stanza/docs/types/bytesize.md @@ -2,7 +2,6 @@ ByteSizes are a type that allows specifying a number of bytes in a human-readable format. See the examples for details. - ## Examples ### Various ways to specify 5000 bytes diff --git a/pkg/stanza/docs/types/entry.md b/pkg/stanza/docs/types/entry.md index 8067d973fbfa0..5053023d60c8f 100644 --- a/pkg/stanza/docs/types/entry.md +++ b/pkg/stanza/docs/types/entry.md @@ -12,7 +12,6 @@ Entry is the base representation of log data as it moves through a pipeline. All | `attributes` | A map of key/value pairs that provide additional context to the log. This value is often used by a consumer to filter logs. | | `body` | The contents of the log. This value is often modified and restructured in the pipeline. It may be a string, number, or object. | - Represented in `json` format, an entry may look like the following: ```json diff --git a/pkg/stanza/docs/types/field.md b/pkg/stanza/docs/types/field.md index b6fb313b8c6d4..c293c4dab5e48 100644 --- a/pkg/stanza/docs/types/field.md +++ b/pkg/stanza/docs/types/field.md @@ -72,7 +72,6 @@ Config: - #### Using fields to refer to various values Given the following entry, we can use fields as follows: diff --git a/pkg/stanza/docs/types/scope_name.md b/pkg/stanza/docs/types/scope_name.md index b9d7e35dedab3..aa2e3325aa12a 100644 --- a/pkg/stanza/docs/types/scope_name.md +++ b/pkg/stanza/docs/types/scope_name.md @@ -10,14 +10,12 @@ Parser operators can parse a scope name and attach the resulting value to a log | --- | --- | --- | | `parse_from` | required | The [field](../types/field.md) from which the value will be parsed. | - ### How to use `scope_name` parsing All parser operators, such as [`regex_parser`](../operators/regex_parser.md) support these fields inside of a `scope_name` block. If a `scope_name` block is specified, the parser operator will perform the parsing _after_ performing its other parsing actions, but _before_ passing the entry to the specified output operator. - ### Example Configurations #### Parse a scope_name from a string diff --git a/pkg/stanza/docs/types/severity.md b/pkg/stanza/docs/types/severity.md index 4859b7e16d9e0..da0be540d20a6 100644 --- a/pkg/stanza/docs/types/severity.md +++ b/pkg/stanza/docs/types/severity.md @@ -145,7 +145,6 @@ The following configurations are equivalent: Additional built-in presets coming soon - ### How to use severity parsing All parser operators, such as [`regex_parser`](../operators/regex_parser.md) support these fields inside of a `severity` block. diff --git a/pkg/stanza/docs/types/timestamp.md b/pkg/stanza/docs/types/timestamp.md index cd50b0a31fdec..88691b2a6c07b 100644 --- a/pkg/stanza/docs/types/timestamp.md +++ b/pkg/stanza/docs/types/timestamp.md @@ -78,7 +78,6 @@ The `epoch` layout type uses can consume epoch-based timestamps. The following l [1] Interpretted as seconds. Equivalent to using `s` layout.
[2] Due to floating point precision limitations, loss of up to 100ns may be expected. - ### How to specify timestamp parsing parameters ```yaml diff --git a/pkg/stanza/docs/types/trace.md b/pkg/stanza/docs/types/trace.md index 8ff21d3e88121..53fdc695acc3f 100644 --- a/pkg/stanza/docs/types/trace.md +++ b/pkg/stanza/docs/types/trace.md @@ -2,7 +2,6 @@ Traces context fields are defined in the [OpenTelemetry Logs Data Model](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#trace-context-fields). - ### `trace` parsing parameters Parser operators can parse a trace context and attach the resulting values to a log entry. @@ -13,7 +12,6 @@ Parser operators can parse a trace context and attach the resulting values to a | `span_id.parse_from` | `span_id` | A [field](../types/field.md) that indicates the field to be parsed as a span ID. | | `trace_flags.parse_from` | `trace_flags` | A [field](../types/field.md) that indicates the field to be parsed as trace flags. | - ### How to use trace parsing All parser operators, such as [`regex_parser`](../operators/regex_parser.md) support these fields inside of a `trace` block. diff --git a/pkg/stanza/fileconsumer/design.md b/pkg/stanza/fileconsumer/design.md index 23100d22cd211..3b09362a8cf9e 100644 --- a/pkg/stanza/fileconsumer/design.md +++ b/pkg/stanza/fileconsumer/design.md @@ -4,13 +4,13 @@ The operator searches the file system for files that meet the following requirem 1. The file's path matches one or more patterns specified in the `include` setting. 2. The file's path does not match any pattern specified in the `exclude` setting. -The set of files that satisfy these requirements are known in this document as "matched files". +The set of files that satisfy these requirements are known in this document as "matched files". The effective search space (`include - exclude`) is referred to colloquially as the operator's "matching pattern". # Fingerprints Files are identified and tracked using fingerprints. A fingerprint is the first `N` bytes of the file, -with the default for `N` being `1000`. +with the default for `N` being `1000`. ### Fingerprint Growth @@ -20,7 +20,7 @@ will be updated, until it reaches the full size of `N`. ### Deduplication of Files -Multiple files with the same fingerprint are handled as if they are the same file. +Multiple files with the same fingerprint are handled as if they are the same file. Most commonly, this circumstance is observed during file rotation that depends on a copy/truncate strategy. After copying the file, but before truncating the original, two files with the same content briefly exist. @@ -50,7 +50,7 @@ pipeline, for example. # Readers -Readers are a convenience struct, which exist for the purpose of managing files and their associated metadata. +Readers are a convenience struct, which exist for the purpose of managing files and their associated metadata. ### Contents @@ -73,14 +73,13 @@ While a file is shorter than the length of a fingerprint, its Reader will contin as it consumes newly written data. A Reader consumes a file using a `bufio.Scanner`, with the Scanner's buffer size defined by the `max_log_size` setting, -and the Scanner's split func defined by the `multiline` setting. +and the Scanner's split func defined by the `multiline` setting. -As each log is read from the file, it is decoded according to the `encoding` function, and then emitted from -the operator. +As each log is read from the file, it is decoded according to the `encoding` function, and then emitted from +the operator. The Reader's offset is updated accordingly whenever a log is emitted. - ### Persistence Readers are always instantiated with an open file handle. Eventually, the file handle is closed, but the Reader is @@ -94,10 +93,9 @@ When the `file_input` operator makes use of a persistence mechanism to save and Setting and Getting a slice of Readers. These Readers contain all the information necessary to pick up exactly where the operator left off. - # Polling -The file system is polled on a regular interval, defined by the `poll_interval` setting. +The file system is polled on a regular interval, defined by the `poll_interval` setting. Each poll cycle runs through a series of steps which are presented below. @@ -158,7 +156,7 @@ Each poll cycle runs through a series of steps which are presented below. - If the file was moved, the open file handle from the previous poll cycle may be useful. 10. Consumption 1. Lost files are consumed. In some cases, such as deletion, this operation will fail. However, if a file - was moved, we may be able to consume the remainder of its content. + was moved, we may be able to consume the remainder of its content. - We do not expect to match this file again, so the best we can do is finish consuming their current contents. - We can reasonably expect in most cases that these files are no longer being written to. 2. Matched files (from this poll cycle) are consumed. @@ -182,8 +180,6 @@ Each poll cycle runs through a series of steps which are presented below. 15. End Poll Cycle 1. At this point, the operator sits idle until the poll timer fires again. - - # Additional Details ### Startup Logic @@ -222,7 +218,6 @@ C) When a file it rotated out of pattern via move/create, we detect that D) When a file it rotated out of pattern via copy/truncate, we detect that our old handle is invalid and we do not attempt to read from it. - #### Rotated files that end up within the matching pattern In both cases of copy/truncate and move/create, if the rotated files match the pattern @@ -247,7 +242,7 @@ logs from the moved file. This can cause data loss. The operator may lose a small percentage of logs, if both of the following conditions are true: 1. The number of files being matched exceeds the maximum degree of concurrency allowed - by the `max_concurrent_files` setting. + by the `max_concurrent_files` setting. 2. Files are being "lost". That is, file rotation is moving files out of the operator's matching pattern, such that subsequent polling cycles will not find these files. diff --git a/pkg/stanza/operator/input/file/design.md b/pkg/stanza/operator/input/file/design.md index cb1678cc69966..7d51ec5ac9c11 100644 --- a/pkg/stanza/operator/input/file/design.md +++ b/pkg/stanza/operator/input/file/design.md @@ -4,12 +4,12 @@ The operator searches the file system for files that meet the following requirem 1. The file's path matches one or more patterns specified in the `include` setting. 2. The file's path does not match any pattern specified in the `exclude` setting. -The set of files that satisfy these requirements are known in this document as "matched files". +The set of files that satisfy these requirements are known in this document as "matched files". The effective search space (`include - exclude`) is referred to colloquially as the operator's "matching pattern". # Fingerprints -Files are identified and tracked using fingerprints. A fingerprint is the first `N` bytes of the file, with the default for `N` being `1000`. +Files are identified and tracked using fingerprints. A fingerprint is the first `N` bytes of the file, with the default for `N` being `1000`. ### Fingerprint Growth @@ -17,7 +17,7 @@ When a file is smaller than `N` bytes, the fingerprint is the entire contents of ### Deduplication of Files -Multiple files with the same fingerprint are handled as if they are the same file. +Multiple files with the same fingerprint are handled as if they are the same file. Most commonly, this circumstance is observed during file rotation that depends on a copy/truncate strategy. After copying the file, but before truncating the original, two files with the same content briefly exist. If the `file_input` operator happens to observe both files at the same time, it will detect a duplicate fingerprint and ingest only one of the files. @@ -27,7 +27,7 @@ In some rare circumstances, a logger may print a very verbose preamble to each l # Readers -Readers are a convenience struct, which exist for the purpose of managing files and their associated metadata. +Readers are a convenience struct, which exist for the purpose of managing files and their associated metadata. ### Contents @@ -46,13 +46,12 @@ Before a Reader begins consuming, it will seek the file's last known offset. If While a file is shorter than the length of a fingerprint, its Reader will continuously append to the fingerprint, as it consumes newly written data. -A Reader consumes a file using a `bufio.Scanner`, with the Scanner's buffer size defined by the `max_log_size` setting, and the Scanner's split func defined by the `multiline` setting. +A Reader consumes a file using a `bufio.Scanner`, with the Scanner's buffer size defined by the `max_log_size` setting, and the Scanner's split func defined by the `multiline` setting. -As each log is read from the file, it is decoded according to the `encoding` function, and then emitted from the operator. +As each log is read from the file, it is decoded according to the `encoding` function, and then emitted from the operator. The Reader's offset is updated accordingly whenever a log is emitted. - ### Persistence Readers are always instantiated with an open file handle. Eventually, the file handle is closed, but the Reader is not immediately discarded. Rather, it is maintained for a fixed number of "poll cycles" (see Polling section below) as a reference to the file's metadata, which may be useful for detecting files that have been moved or copied, and for recalling metadata such as the file's previous path. @@ -61,10 +60,9 @@ Readers are maintained for a fixed period of time, and then discarded. When the `file_input` operator makes use of a persistence mechanism to save and recall its state, it is simply Setting and Getting a slice of Readers. These Readers contain all the information necessary to pick up exactly where the operator left off. - # Polling -The file system is polled on a regular interval, defined by the `poll_interval` setting. +The file system is polled on a regular interval, defined by the `poll_interval` setting. Each poll cycle runs through a series of steps which are presented below. @@ -109,7 +107,7 @@ Each poll cycle runs through a series of steps which are presented below. - The file may have been rotated to another location. - If the file was moved, the open file handle from the previous poll cycle may be useful. 10. Consumption - 1. Lost files are consumed. In some cases, such as deletion, this operation will fail. However, if a file was moved, we may be able to consume the remainder of its content. + 1. Lost files are consumed. In some cases, such as deletion, this operation will fail. However, if a file was moved, we may be able to consume the remainder of its content. - We do not expect to match this file again, so the best we can do is finish consuming their current contents. - We can reasonably expect in most cases that these files are no longer being written to. 2. Matched files (from this poll cycle) are consumed. @@ -129,8 +127,6 @@ Each poll cycle runs through a series of steps which are presented below. 15. End Poll Cycle 1. At this point, the operator sits idle until the poll timer fires again. - - # Additional Details ### Startup Logic @@ -149,13 +145,12 @@ When the operator shuts down, the following occurs: The net effect of the shut down routine is that all files are checkpointed in a normal manner (i.e. not in the middle of a log entry), and all checkpoints are persisted. - # Known Limitations ### Potential data loss when maximum concurrency must be enforced The operator may lose a small percentage of logs, if both of the following conditions are true: -1. The number of files being matched exceeds the maximum degree of concurrency allowed by the `max_concurrent_files` setting. +1. The number of files being matched exceeds the maximum degree of concurrency allowed by the `max_concurrent_files` setting. 2. Files are being "lost". That is, file rotation is moving files out of the operator's matching pattern, such that subsequent polling cycles will not find these files. When both of these conditions occur, it is impossible for the operator to both: diff --git a/pkg/translator/azurelogs/README.md b/pkg/translator/azurelogs/README.md index 875b37ba5e5bf..4889aba891c53 100644 --- a/pkg/translator/azurelogs/README.md +++ b/pkg/translator/azurelogs/README.md @@ -33,7 +33,6 @@ https://learn.microsoft.com/en-us/azure/azure-monitor/platform/resource-logs-sch | `correlationId` | `azure.correlation.id` | | `identity` | `azure.identity` | - ### Identity | Original Field (JSON) | Log Record Attribute | diff --git a/pkg/xstreamencoding/README.md b/pkg/xstreamencoding/README.md index 5033256803e60..6888483a83829 100644 --- a/pkg/xstreamencoding/README.md +++ b/pkg/xstreamencoding/README.md @@ -3,7 +3,7 @@ > [!NOTE] > These helpers are experimental and may change in future releases. -This package provides reusable helpers for stream-based unmarshaling of OpenTelemetry signals. +This package provides reusable helpers for stream-based unmarshaling of OpenTelemetry signals. It is designed to support efficient processing of **newline-delimited** streams with configurable batching and flushing behavior. ## Overview @@ -125,7 +125,7 @@ func (c *myCodec) NewLogsDecoder(reader io.Reader, options ...encoding.DecoderOp if err != nil { return nil, err } - + logs := plog.NewLogs() decodeFunc := func() (plog.Logs, error) { @@ -160,4 +160,3 @@ func (c *myCodec) NewLogsDecoder(reader io.Reader, options ...encoding.DecoderOp return xstreamencoding.NewLogsDecoderAdapter(decodeFunc, offsetFunc), nil } ``` - diff --git a/processor/attributesprocessor/README.md b/processor/attributesprocessor/README.md index d508d9fd65ddf..2429580214786 100644 --- a/processor/attributesprocessor/README.md +++ b/processor/attributesprocessor/README.md @@ -90,7 +90,6 @@ For the `delete` action, pattern: ``` - For the `hash` action, - `key` and/or `pattern` is required - `action: hash` is required. @@ -102,7 +101,6 @@ For the `hash` action, pattern: ``` - For the `extract` action, - `key` is required - `pattern` is required. @@ -119,7 +117,6 @@ For the `extract` action, ``` - For the `convert` action, - `key` is required - `action: convert` is required. @@ -208,21 +205,20 @@ Metric transform processor specific functionality ## Include/Exclude Filtering -The [attribute processor](README.md) exposes an option to provide a set of properties of a span, log +The [attribute processor](README.md) exposes an option to provide a set of properties of a span, log or metric record to match against to determine if the input data should be included or excluded from -the processor. To configure this option, under `include` and/or `exclude` at least `match_type` and +the processor. To configure this option, under `include` and/or `exclude` at least `match_type` and one of the following is required: -- For spans, one of `services`, `span_names`, `span_kinds`, `attributes`, `resources` or `libraries` -must be specified with a non-empty value for a valid configuration. The `log_bodies`, `log_severity_texts`, +- For spans, one of `services`, `span_names`, `span_kinds`, `attributes`, `resources` or `libraries` +must be specified with a non-empty value for a valid configuration. The `log_bodies`, `log_severity_texts`, `log_severity_number` and `metric_names` fields are invalid. - For logs, one of `log_bodies`, `log_severity_texts`, `log_severity_number`, `attributes`, `resources` -or `libraries` must be specified with a non-empty value for a valid configuration. The `span_names`, +or `libraries` must be specified with a non-empty value for a valid configuration. The `span_names`, `span_kinds`, `metric_names` and `services` fields are invalid. - For metrics, one of `metric_names` or `resources` must be specified with a valid non-empty value for a valid configuration. The `span_names`, `span_kinds`, `log_bodies`, `log_severity_texts`, `log_severity_number`, `services`, `attributes` and `libraries` fields are invalid. - Note: If both `include` and `exclude` are specified, the `include` properties are checked before the `exclude` properties. diff --git a/processor/cardinalityguardianprocessor/README.md b/processor/cardinalityguardianprocessor/README.md index ba61c5079e7bb..09611a1b193c8 100644 --- a/processor/cardinalityguardianprocessor/README.md +++ b/processor/cardinalityguardianprocessor/README.md @@ -56,7 +56,7 @@ processors: ## Warnings Care needs to be taken when modifying data point attributes using this processor: -- **Single-Writer Conflict**: In enforcement mode (`tag_only: false`), the processor strips attributes. This violates the OTel metrics single-writer rule. When multiple data points collapse into the same timeseries identity, backends like Prometheus will interpret the overlapping values as counter resets, producing silently incorrect `rate()` and `increase()` results. This affects all cumulative Sum and Histogram metrics where enforcement fires. +- **Single-Writer Conflict**: In enforcement mode (`tag_only: false`), the processor strips attributes. This violates the OTel metrics single-writer rule. When multiple data points collapse into the same timeseries identity, backends like Prometheus will interpret the overlapping values as counter resets, producing silently incorrect `rate()` and `increase()` results. This affects all cumulative Sum and Histogram metrics where enforcement fires. Use `tag_only: true` with a routing processor for production safety until a downstream spatial reaggregation processor is available. ## Troubleshooting diff --git a/processor/cumulativetodeltaprocessor/README.md b/processor/cumulativetodeltaprocessor/README.md index 912fd78761d40..c3d375fd52772 100644 --- a/processor/cumulativetodeltaprocessor/README.md +++ b/processor/cumulativetodeltaprocessor/README.md @@ -5,7 +5,6 @@ The cumulative to delta processor (`cumulativetodeltaprocessor`) converts monoto histogram metrics from cumulative to delta aggregation temporality. Non-monotonic sums are excluded. Delta metrics are excluded from any conversion and forwarded without changes. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/processor/deltatocumulativeprocessor/README.md b/processor/deltatocumulativeprocessor/README.md index d863992085537..c8f351a4edd55 100644 --- a/processor/deltatocumulativeprocessor/README.md +++ b/processor/deltatocumulativeprocessor/README.md @@ -4,7 +4,6 @@ The Delta to Cumulative Processor (`deltatocumulativeprocessor`) converts metrics from delta temporality to cumulative, by accumulating samples in memory. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -39,4 +38,4 @@ There is no further configuration required. All delta samples are converted to c ## Troubleshooting When [Telemetry is -enabled](https://opentelemetry.io/docs/collector/configuration/#telemetry), this component exports [several metrics](./documentation.md). +enabled](https://opentelemetry.io/docs/collector/configuration/#telemetry), this component exports [several metrics](./documentation.md). diff --git a/processor/deltatorateprocessor/README.md b/processor/deltatorateprocessor/README.md index efc4dcb79e57a..743ffcaf4997f 100644 --- a/processor/deltatorateprocessor/README.md +++ b/processor/deltatorateprocessor/README.md @@ -1,8 +1,7 @@ # Delta to Rate Processor -The Delta to Rate Processor (`deltatorateprocessor`) converts delta sum metrics to rate metrics. This rate is a gauge. - +The Delta to Rate Processor (`deltatorateprocessor`) converts delta sum metrics to rate metrics. This rate is a gauge. | Status | | | ------------- |-----------| diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index 6d388be9ab2ef..6154a72ef03b7 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -107,7 +107,7 @@ The filter processor also allows configuring an optional field, `error_mode`, wh ##### `processor.filter.defaultErrorModeIgnore` -The `processor.filter.defaultErrorModeIgnore` feature gate changes the default `error_mode` of the filter processor from `propagate` to `ignore`. +The `processor.filter.defaultErrorModeIgnore` feature gate changes the default `error_mode` of the filter processor from `propagate` to `ignore`. `ignore` is the recommended mode to improve resiliency, as errors are logged for visibility but valid data is preserved, and processing continues with the next condition. This feature gate is currently in Alpha (disabled by default) and must be explicitly enabled. @@ -254,7 +254,7 @@ trace_conditions: - IsRootSpan() or spanevent.name == "bar" ``` -This condition fails because `IsRootSpan()` is only available in the span context, not span events, +This condition fails because `IsRootSpan()` is only available in the span context, not span events, while the `spanevent` Path prefix requires the condition to evaluate in the span event context. The solution is to split into separate span and span event conditions: @@ -268,7 +268,7 @@ trace_conditions: - spanevent.name == "bar" ``` -The advanced configuration is required for `IsRootSpan()` because there is no Path prefix in the condition, +The advanced configuration is required for `IsRootSpan()` because there is no Path prefix in the condition, so the context must be set explicitly to `span`. ### Examples diff --git a/processor/geoipprocessor/README.md b/processor/geoipprocessor/README.md index f8115e91f10ae..99d9683893199 100644 --- a/processor/geoipprocessor/README.md +++ b/processor/geoipprocessor/README.md @@ -4,7 +4,6 @@ The geoIP processor `geoipprocessor` enhances the attributes of a span, log, or metric by appending information about the geographical location of an IP address. - | Status | | | ------------- |-----------| | Stability | [alpha]: traces, metrics, logs | diff --git a/processor/groupbyattrsprocessor/README.md b/processor/groupbyattrsprocessor/README.md index e7965ac4be61d..d4bc75b77477d 100644 --- a/processor/groupbyattrsprocessor/README.md +++ b/processor/groupbyattrsprocessor/README.md @@ -5,7 +5,6 @@ This processor re-associates spans, log records and metric datapoints to a *Reso attributes. As a result, all spans, log records or metric datapoints with the same values for the specified attributes are "grouped" under the same *Resource*. - | Status | | | ------------- |-----------| | Stability | [beta]: traces, metrics, logs | @@ -133,7 +132,7 @@ Resource {host.name="otherhost"} Span {span_id=5, ...} ``` -With the below configuration, the **groupbyattrs** will re-associate the spans with matching Resource and InstrumentationLibrary. +With the below configuration, the **groupbyattrs** will re-associate the spans with matching Resource and InstrumentationLibrary. ```yaml processors: diff --git a/processor/groupbytraceprocessor/README.md b/processor/groupbytraceprocessor/README.md index cbcc2d8284f0f..9a598677576a4 100644 --- a/processor/groupbytraceprocessor/README.md +++ b/processor/groupbytraceprocessor/README.md @@ -15,15 +15,15 @@ [k8s]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-k8s -This processor collects all the spans from the same trace, waiting a +This processor collects all the spans from the same trace, waiting a pre-determined amount of time before releasing the trace to the next processor. The expectation is that, generally, traces will be complete after the given time. This processor should be used whenever a processor requires grouped traces to make decisions, -such as a tail-based sampler or a per-trace metrics processor. Note that [`tailsamplingprocessor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) +such as a tail-based sampler or a per-trace metrics processor. Note that [`tailsamplingprocessor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) also implements a similar mechanism and can be used independently. -The batch processor shouldn't be used before this processor, as this one will +The batch processor shouldn't be used before this processor, as this one will probably undo part (or much) of the work that the batch processor performs. It's fine to have the batch processor to run right after this one, and every entry in the batch will be a complete trace. @@ -50,7 +50,7 @@ The `num_traces` (default=1,000,000) property tells the processor what's the max The `wait_duration` (default=1s) property tells the processor for how long it should keep traces in the internal storage. Once a trace is kept for this duration, it's then released to the next consumer and removed from the internal storage. Spans from a trace that has been released will be kept for the entire duration again. The `num_workers` (default=1) property controls how many concurrent workers the processor will use to process traces. If you are looking to optimize this value -then using GOMAXPROCS could be considered as a starting point. +then using GOMAXPROCS could be considered as a starting point. ## Metrics @@ -78,4 +78,4 @@ Most metrics are updated when the events occur, except for the following ones, w ## Warnings -- [Statefulness](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/standard-warnings.md#statefulness): The groupbytrace processor's works best when all spans for a trace are sent to the same collector instance. +- [Statefulness](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/standard-warnings.md#statefulness): The groupbytrace processor's works best when all spans for a trace are sent to the same collector instance. diff --git a/processor/intervalprocessor/README.md b/processor/intervalprocessor/README.md index 761754eb5afeb..ce6926ed9423e 100644 --- a/processor/intervalprocessor/README.md +++ b/processor/intervalprocessor/README.md @@ -22,7 +22,7 @@ The interval processor (`intervalprocessor`) aggregates metrics and periodically * Monotonically increasing, cumulative sums * Monotonically increasing, cumulative histograms * Monotonically increasing, cumulative exponential histograms -* Gauges +* Gauges * Summaries The following metric types will *not* be aggregated, and will instead be passed, unchanged, to the next component in the pipeline: diff --git a/processor/isolationforestprocessor/README.md b/processor/isolationforestprocessor/README.md index 7299799c5777b..d3f54d8a91b8a 100644 --- a/processor/isolationforestprocessor/README.md +++ b/processor/isolationforestprocessor/README.md @@ -6,7 +6,6 @@ The **Isolation Forest processor** adds inline, unsupervised anomaly detection t learns normal behaviour from recent telemetry and tags, scores, or optionally drops anomalies *in‑flight* – no external ML service required. - | Status | | | ------------- |-----------| | Stability | [alpha]: traces, metrics, logs | @@ -136,7 +135,7 @@ service: ``` > Note: Use `routingconnector` to seggregate the different kind of spans(db, messaging etc.) and send them to separate `isolationforestprocessor` deployments so the anomaly detection is pertianing to the respective category of signals. - + ### What the example does | Signal | What’s scored | Feature grouping | Output | Notes | @@ -171,13 +170,11 @@ Telemetry ───▶ │ • Score calculator & anomaly decision └───────────────────────────────────────────────────┘ ``` - *Training cost*: **O(current_window_size × forest_size × log subsample_size)** every `training_interval` *Scoring cost*: **O(forest_size × log subsample_size)** per item **Note:** With adaptive window sizing enabled, `current_window_size` dynamically adjusts between `min_window_size` and `max_window_size` based on traffic patterns and memory constraints, making training costs adaptive to workload conditions. - --- ## 🤝 Contributing @@ -192,4 +189,3 @@ Telemetry ───▶ │ • Score calculator & anomaly decision PRs welcome – please include unit tests and doc updates. --- - diff --git a/processor/k8sattributesprocessor/README.md b/processor/k8sattributesprocessor/README.md index 620c0782f946a..fed6ffaa7b140 100644 --- a/processor/k8sattributesprocessor/README.md +++ b/processor/k8sattributesprocessor/README.md @@ -3,7 +3,6 @@ The Kubernetes Attributes Processor allows adding k8s metadata to resource attributes for spans, metrics and logs. - | Status | | | ------------- |-----------| | Stability | [development]: profiles | @@ -233,7 +232,6 @@ Because resource state modifications are already pushed immediately via Kubernet - `watch_sync_period` (`default: 5m`): The resync period for K8s informers. You may set this to `0s` to disable resyncing completely (recommended for large clusters). - ## Extracting attributes from pod labels and annotations The k8sattributesprocessor can also set resource attributes from k8s labels and annotations of pods, namespaces, deployments, statefulsets, daemonsets, jobs and nodes. @@ -1140,9 +1138,9 @@ are currently in `beta` stability and are actively moving towards `stable` stabi ## Available Benchmarks The component is tested as part of the project's load tests, with the results being publicly -available at the benchmarks +available at the benchmarks [page](https://open-telemetry.github.io/opentelemetry-collector-contrib/benchmarks/loadtests). -In that page, users can find details such as +In that page, users can find details such as [memory](https://open-telemetry.github.io/opentelemetry-collector-contrib/benchmarks/loadtests/#metrick8sattributesprocessor-5k-workload-cluster-ram-mib) and [CPU](https://open-telemetry.github.io/opentelemetry-collector-contrib/benchmarks/loadtests/#metrick8sattributesprocessor-5k-workload-cluster-cpu-percentage) performance when the component is used in K8s Clusters (tests use [KWOK](https://kwok.sigs.k8s.io/)) diff --git a/processor/logdedupprocessor/README.md b/processor/logdedupprocessor/README.md index bf55abf2507df..ef4dcbe231113 100644 --- a/processor/logdedupprocessor/README.md +++ b/processor/logdedupprocessor/README.md @@ -4,7 +4,6 @@ The Log DeDuplication Processor is used to deduplicate logs by detecting identical logs over a range of time and emitting a single log with the count of logs that were deduplicated. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | @@ -36,7 +35,7 @@ emitting a single log with the count of logs that were deduplicated. | interval | duration | `10s` | The interval at which logs are aggregated. The counter will reset after each interval. | | conditions | []string | `[]` | A slice of [OTTL] expressions used to evaluate which log records are deduped. All paths in the [log context] are available to reference. Paths should be prefixed with their context name (e.g. `log.attributes["foo"]`, `resource.attributes["bar"]`). The un-prefixed form (e.g. `attributes["foo"]`) is deprecated; if used, the processor will log the rewritten conditions at startup so they can be migrated. All [converters] are available to use. | | log_count_attribute | string | `log_count` | The name of the count attribute of deduplicated logs that will be added to the emitted aggregated log. | -| include_fields | []string | `[]` | Fields to include in duplication matching. Fields can be from the log `body` or `attributes`. Nested fields must be `.` delimited. If a field contains a `.` it can be escaped by using a `\`. This option is **mutually exclusive** with `exclude_fields`. See [example config](#example-config-with-deduplication-key). +| include_fields | []string | `[]` | Fields to include in duplication matching. Fields can be from the log `body` or `attributes`. Nested fields must be `.` delimited. If a field contains a `.` it can be escaped by using a `\`. This option is **mutually exclusive** with `exclude_fields`. See [example config](#example-config-with-deduplication-key). | | timezone | string | `UTC` | The timezone of the `first_observed_timestamp` and `last_observed_timestamp` timestamps on the emitted aggregated log. The available locations depend on the local IANA Time Zone database. [This page](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) contains many examples, such as `America/New_York`. | | exclude_fields | []string | `[]` | Fields to exclude from duplication matching. Fields can be excluded from the log `body` or `attributes`. These fields will not be present in the emitted aggregated log. Nested fields must be `.` delimited. This option is `mutually exclusive` with `include_fields`. If a field contains a `.` it can be escaped by using a `\` see [example config](#example-config-with-excluded-fields).

**Note**: The entire `body` cannot be excluded. If the body is a map then fields within it can be excluded. | diff --git a/processor/lookupprocessor/README.md b/processor/lookupprocessor/README.md index 89012a00f3928..134ba5032380f 100644 --- a/processor/lookupprocessor/README.md +++ b/processor/lookupprocessor/README.md @@ -7,7 +7,6 @@ expression to extract a lookup key, queries a lookup source, and writes the resu Currently supports logs, with metrics and traces support planned. - | Status | | | ------------- |-----------| | Stability | [development]: logs | diff --git a/processor/metricsgenerationprocessor/README.md b/processor/metricsgenerationprocessor/README.md index ed72f9f8ec35b..edc355183ed78 100644 --- a/processor/metricsgenerationprocessor/README.md +++ b/processor/metricsgenerationprocessor/README.md @@ -35,7 +35,7 @@ There are some specific behaviors of the `calculate` metric generation rule that ## Configuration -Configuration is specified through a list of generation rules. Generation rules find the metrics which +Configuration is specified through a list of generation rules. Generation rules find the metrics which match the given metric names and apply the specified operation to those metrics. ```yaml diff --git a/processor/metricstarttimeprocessor/README.md b/processor/metricstarttimeprocessor/README.md index 436b1792f71b5..104f08b142233 100644 --- a/processor/metricstarttimeprocessor/README.md +++ b/processor/metricstarttimeprocessor/README.md @@ -6,7 +6,6 @@ time for metric points with a cumulative aggregation temporality. It is commonly used with the `prometheus` receiver, which usually produces metric points without a start time. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/processor/metricstransformprocessor/README.md b/processor/metricstransformprocessor/README.md index d9e22377d84ba..b616782adbef6 100644 --- a/processor/metricstransformprocessor/README.md +++ b/processor/metricstransformprocessor/README.md @@ -307,7 +307,7 @@ operations: ... ``` -### Group Metrics +### Group Metrics ```yaml # Group metrics from one single ResourceMetrics and report them as multiple ResourceMetrics. # diff --git a/processor/probabilisticsamplerprocessor/README.md b/processor/probabilisticsamplerprocessor/README.md index 9be1166214b3d..af663ffe275f5 100644 --- a/processor/probabilisticsamplerprocessor/README.md +++ b/processor/probabilisticsamplerprocessor/README.md @@ -184,7 +184,7 @@ for every 4 items input. This mode uses the same randomness mechanism as the proportional sampling mode, in this case considering how much each item was already sampled by preceding samplers. This mode can be used to lower -sampling probability to a minimum value across a whole pipeline, +sampling probability to a minimum value across a whole pipeline, making it possible to conditionally adjust sampling probabilities. This mode compares a 56 bit threshold against the configured sampling diff --git a/processor/redactionprocessor/README.md b/processor/redactionprocessor/README.md index ba81fd26da9ba..4f55ffd405158 100644 --- a/processor/redactionprocessor/README.md +++ b/processor/redactionprocessor/README.md @@ -6,7 +6,6 @@ attributes. It also masks attribute values that match a blocked value list. Attributes that aren't on the allowed list are removed before any value checks are done. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs, metrics | @@ -158,7 +157,6 @@ processors: ``` - `blocked_key_patterns` applies to the values of the keys matching one of the patterns. The value is then masked according to the configuration. diff --git a/processor/remotetapprocessor/README.md b/processor/remotetapprocessor/README.md index 2c965f6cabcbd..3d94901719376 100644 --- a/processor/remotetapprocessor/README.md +++ b/processor/remotetapprocessor/README.md @@ -19,7 +19,7 @@ of the data accessible to WebSocket clients connecting on a configurable port. This functionality resembles that of the Unix `tee` command, which enables data to flow through while duplicating and redirecting it for inspection. -To avoid overloading clients, the amount of telemetry duplicated over +To avoid overloading clients, the amount of telemetry duplicated over any open WebSockets is rate limited by an adjustable amount. ## Config diff --git a/processor/resourcedetectionprocessor/README.md b/processor/resourcedetectionprocessor/README.md index fb2e7a8d0b4d0..a77bc2a260b08 100644 --- a/processor/resourcedetectionprocessor/README.md +++ b/processor/resourcedetectionprocessor/README.md @@ -176,27 +176,27 @@ The list of the populated resource attributes can be found at [GCP Detector Reso #### GCE Metadata - * cloud.provider ("gcp") - * cloud.platform ("gcp_compute_engine") - * cloud.account.id (project id) - * cloud.region (e.g. us-central1) - * cloud.availability_zone (e.g. us-central1-c) - * host.id (instance id) - * host.name (instance name) - * host.type (machine type) - * (optional) gcp.gce.instance.hostname - * (optional) gcp.gce.instance.name +- cloud.provider ("gcp") +- cloud.platform ("gcp_compute_engine") +- cloud.account.id (project id) +- cloud.region (e.g. us-central1) +- cloud.availability_zone (e.g. us-central1-c) +- host.id (instance id) +- host.name (instance name) +- host.type (machine type) +- (optional) gcp.gce.instance.hostname +- (optional) gcp.gce.instance.name #### GKE Metadata - * cloud.provider ("gcp") - * cloud.platform ("gcp_kubernetes_engine") - * cloud.account.id (project id) - * cloud.region (only for regional GKE clusters; e.g. "us-central1") - * cloud.availability_zone (only for zonal GKE clusters; e.g. "us-central1-c") - * k8s.cluster.name - * host.id (instance id) - * host.name (instance name; only when workload identity is disabled) +- cloud.provider ("gcp") +- cloud.platform ("gcp_kubernetes_engine") +- cloud.account.id (project id) +- cloud.region (only for regional GKE clusters; e.g. "us-central1") +- cloud.availability_zone (only for zonal GKE clusters; e.g. "us-central1-c") +- k8s.cluster.name +- host.id (instance id) +- host.name (instance name; only when workload identity is disabled) One known issue is when GKE workload identity is enabled, the GCE metadata endpoints won't be available, thus the GKE resource detector won't be able to determine `host.name`. In that case, users are encouraged to set `host.name` from either: @@ -205,45 +205,45 @@ able to determine `host.name`. In that case, users are encouraged to set `host.n #### Google Cloud Run Services Metadata - * cloud.provider ("gcp") - * cloud.platform ("gcp_cloud_run") - * cloud.account.id (project id) - * cloud.region (e.g. "us-central1") - * faas.instance (instance id) - * faas.name (service name) - * faas.version (service revision) +- cloud.provider ("gcp") +- cloud.platform ("gcp_cloud_run") +- cloud.account.id (project id) +- cloud.region (e.g. "us-central1") +- faas.instance (instance id) +- faas.name (service name) +- faas.version (service revision) #### Cloud Run Jobs Metadata - * cloud.provider ("gcp") - * cloud.platform ("gcp_cloud_run") - * cloud.account.id (project id) - * cloud.region (e.g. "us-central1") - * faas.instance (instance id) - * faas.name (service name) - * gcp.cloud_run.job.execution ("my-service-ajg89") - * gcp.cloud_run.job.task_index ("0") +- cloud.provider ("gcp") +- cloud.platform ("gcp_cloud_run") +- cloud.account.id (project id) +- cloud.region (e.g. "us-central1") +- faas.instance (instance id) +- faas.name (service name) +- gcp.cloud_run.job.execution ("my-service-ajg89") +- gcp.cloud_run.job.task_index ("0") #### Google Cloud Functions Metadata - * cloud.provider ("gcp") - * cloud.platform ("gcp_cloud_functions") - * cloud.account.id (project id) - * cloud.region (e.g. "us-central1") - * faas.instance (instance id) - * faas.name (function name) - * faas.version (function version) +- cloud.provider ("gcp") +- cloud.platform ("gcp_cloud_functions") +- cloud.account.id (project id) +- cloud.region (e.g. "us-central1") +- faas.instance (instance id) +- faas.name (function name) +- faas.version (function version) #### Google App Engine Metadata - * cloud.provider ("gcp") - * cloud.platform ("gcp_app_engine") - * cloud.account.id (project id) - * cloud.region (e.g. "us-central1") - * cloud.availability_zone (e.g. "us-central1-c") - * faas.instance (instance id) - * faas.name (service name) - * faas.version (service version) +- cloud.provider ("gcp") +- cloud.platform ("gcp_app_engine") +- cloud.account.id (project id) +- cloud.region (e.g. "us-central1") +- cloud.availability_zone (e.g. "us-central1-c") +- faas.instance (instance id) +- faas.name (service name) +- faas.version (service version) ### AWS EC2 @@ -453,7 +453,7 @@ processors: Matched tags are added as: - * azure.tags. +- azure.tags.\ ### Azure AKS diff --git a/processor/schemaprocessor/DESIGN.md b/processor/schemaprocessor/DESIGN.md index 802bed0cf4589..9879b8fb47e01 100644 --- a/processor/schemaprocessor/DESIGN.md +++ b/processor/schemaprocessor/DESIGN.md @@ -18,11 +18,11 @@ graph LR; F --> G[Migrator] end -``` +``` The [Schema Processor](processor.go) is registered as a Processor in the Collector by the factory. Data flows into the Transformer, which uses the Schema URL to fetch the translation from the Translation Manager. The Translation Manager (at internal/translation/manager.go in a future PR) is responsible for fetching and caching the translations. It takes in a schema URL and returns a Translator struct. -The Translator struct contains the target schema URL, the target schema version, and a list of Revisions. The Translator figures out what the version of the incoming data is and what Revisions to apply to the incoming data to get it to the target schema version. The Translator is also responsible for applying the Revisions to the incoming data - it iterates through these Revisions and applies them to the incoming data. +The Translator struct contains the target schema URL, the target schema version, and a list of Revisions. The Translator figures out what the version of the incoming data is and what Revisions to apply to the incoming data to get it to the target schema version. The Translator is also responsible for applying the Revisions to the incoming data - it iterates through these Revisions and applies them to the incoming data. Each Revision represents all the changes within a specific version. It consists of several ChangeLists (at internal/changelist/changelist.go in a future PR) - one for each type of change block (at the time of writing - `all`, `resources`, `spans`, `spanEvents`, `metrics`, `logs`). Each ChangeList is similar to a program in an interpreter - in this case the programming language is the schema file! They iterate through whatever changes they are constructed with, and call a [Transformer](internal/transformer) for each type of change. The Transformer accepts a typed value - a log, a metric, etc. It then, under the hood, calls one of a few Migrators. The Migrators do the fundamental work of changing attributes, changing names, etc. The Migrators generally operate on lower levels than the Transformers - they operate on `Attributes`, or an `alias.NamedSignal` (a signal that implements `Name()` and `SetName()`). diff --git a/processor/schemaprocessor/GUIDE.md b/processor/schemaprocessor/GUIDE.md index dc3631ac059eb..bd66d2088bcf4 100644 --- a/processor/schemaprocessor/GUIDE.md +++ b/processor/schemaprocessor/GUIDE.md @@ -16,7 +16,7 @@ This creates two problems: - **Collector pipeline** — processors downstream of the schema processor that filter, route, or transform based on attribute names (e.g., `filterprocessor`, `transformprocessor`, `routingconnector`) - **Backend queries and dashboards** — any saved queries or dashboard panels referencing old names - **Alerts** — monitoring rules that depend on specific attribute names - + There's no error — just missing data. This discourages operators from updating their target version, even as their services naturally adopt newer instrumentation. The schema processor solves both problems: it **normalizes all telemetry to a single version** regardless of what each service emits, and the migration feature lets you **transition between versions safely** with both old and new names preserved during the changeover. diff --git a/processor/spanprocessor/README.md b/processor/spanprocessor/README.md index 742e230dcddd2..49990944e64ed 100644 --- a/processor/spanprocessor/README.md +++ b/processor/spanprocessor/README.md @@ -81,7 +81,7 @@ span name that is the output after processing the previous rule. - `break_after_match` (default = false): specifies if processing of rules should stop after the first match. If it is false rule processing will continue to be performed over the modified span name. -- `keep_original_name` (default = false): specifies if the original span name should be kept after +- `keep_original_name` (default = false): specifies if the original span name should be kept after processing the rules. If it is true, the original span name will be kept, otherwise it will be replaced with the placeholders of the captured attributes. diff --git a/processor/tailsamplingprocessor/README.md b/processor/tailsamplingprocessor/README.md index 24dd8bad697d4..405242495da91 100644 --- a/processor/tailsamplingprocessor/README.md +++ b/processor/tailsamplingprocessor/README.md @@ -669,7 +669,7 @@ When this feature gate is set, this will add additional attributes on each sampl ### Tail storage extension -To configure `tail_storage` on the tailsampling processor, you must enable the `processor.tailsamplingprocessor.tailstorageextension` feature gate. +To configure `tail_storage` on the tailsampling processor, you must enable the `processor.tailsamplingprocessor.tailstorageextension` feature gate. When a storage extension implements the experimental `TailStorage` extension, it will be used instead of the default in-memory approach. diff --git a/processor/transformprocessor/README.md b/processor/transformprocessor/README.md index 03877206a1316..5425ef6148b66 100644 --- a/processor/transformprocessor/README.md +++ b/processor/transformprocessor/README.md @@ -121,8 +121,8 @@ transform: - set(profile.original_payload_format, "json") ``` -In some situations a combination of Paths, functions, or enums is not allowed, and the solution -might require multiple [Advanced Config](#advanced-config) configuration groups. +In some situations a combination of Paths, functions, or enums is not allowed, and the solution +might require multiple [Advanced Config](#advanced-config) configuration groups. See [Context Inference](#context-inference) for more details. ### Advanced Config @@ -182,7 +182,7 @@ transform: ``` The Transform Processor will enforce that all the Paths, functions, and enums used in a group's `statements` are parsable. -In some situations a combination of Paths, functions, or enums is not allowed, and it might require multiple configuration groups. +In some situations a combination of Paths, functions, or enums is not allowed, and it might require multiple configuration groups. See [Context Inference](#context-inference) for more details. ### Context inference @@ -295,7 +295,7 @@ Examples: Converts incoming metrics of type "Gauge" to type "Sum", retaining the metric's datapoints and setting its aggregation temporality and monotonicity accordingly. Noop for metrics that are not of type "Gauge". -`aggregation_temporality` is a string (`"cumulative"` or `"delta"`) that specifies the resultant metric's aggregation temporality. `is_monotonic` is a boolean that specifies the resultant metric's monotonicity. +`aggregation_temporality` is a string (`"cumulative"` or `"delta"`) that specifies the resultant metric's aggregation temporality. `is_monotonic` is a boolean that specifies the resultant metric's monotonicity. **NOTE:** This function may cause a metric to break semantics for [Sum metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md#sums). Use at your own risk. @@ -303,7 +303,6 @@ Examples: - `convert_gauge_to_sum("cumulative", false)` - - `convert_gauge_to_sum("delta", true)` ### extract_count_metric @@ -463,13 +462,11 @@ Examples: - `copy_metric(name="http.request.status_code", unit="s") where metric.name == "http.status_code` - - `copy_metric(desc="new desc") where metric.description == "old desc"` - ### convert_exponential_histogram_to_histogram -__Warning:__ The approach used in this function to convert exponential histograms to explicit histograms __is not__ part of the __OpenTelemetry Specification__. +**Warning:** The approach used in this function to convert exponential histograms to explicit histograms **is not** part of the **OpenTelemetry Specification**. `convert_exponential_histogram_to_histogram(distribution, [ExplicitBounds])` @@ -479,7 +476,7 @@ This function requires 2 arguments: - `distribution` - This argument defines the distribution algorithm used to allocate the exponential histogram datapoints into a new Explicit Histogram. There are 4 options: - - __upper__ - This approach identifies the highest possible value of each exponential bucket (_the upper bound_) and uses it to distribute the datapoints by comparing the upper bound of each bucket with the ExplicitBounds provided. This approach works better for small/narrow exponential histograms where the difference between the upper bounds and lower bounds are small. + - **upper** - This approach identifies the highest possible value of each exponential bucket (_the upper bound_) and uses it to distribute the datapoints by comparing the upper bound of each bucket with the ExplicitBounds provided. This approach works better for small/narrow exponential histograms where the difference between the upper bounds and lower bounds are small. _For example, Given:_ 1. count = 10 @@ -491,58 +488,55 @@ This function requires 2 arguments: - $15>5$ (_skip_) - $15>10$ (_skip_) - $15<=15$ (allocate count to this boundary) - 6. Allocate count: [0, 0, __10__, 0, 0] - 7. Final Counts: [0, 0, __10__, 0, 0] + 6. Allocate count: [0, 0, **10**, 0, 0] + 7. Final Counts: [0, 0, **10**, 0, 0] - - __midpoint__ - This approach works in a similar way to the __upper__ approach, but instead of using the upper bound, it uses the midpoint of each exponential bucket. The midpoint is identified by calculating the average of the upper and lower bounds. This approach also works better for small/narrow exponential histograms. + - **midpoint** - This approach works in a similar way to the **upper** approach, but instead of using the upper bound, it uses the midpoint of each exponential bucket. The midpoint is identified by calculating the average of the upper and lower bounds. This approach also works better for small/narrow exponential histograms. - - >The __uniform__ and __random__ distribution algorithms both utilise the concept of intersecting boundaries. - Intersecting boundaries are any boundary in the `boundaries array` that falls between or on the lower and upper values of the Exponential Histogram boundaries. + >The **uniform** and **random** distribution algorithms both utilise the concept of intersecting boundaries. + Intersecting boundaries are any boundary in the `boundaries array` that falls between or on the lower and upper values of the Exponential Histogram boundaries. _For Example:_ if you have an Exponential Histogram bucket with a lower bound of 10 and upper of 20, and your boundaries array is [5, 10, 15, 20, 25], the intersecting boundaries are 10, 15, and 20 because they lie within the range [10, 20]. - - __uniform__ - This approach distributes the datapoints for each bucket uniformly across the intersecting __ExplicitBounds__. The algorithm works as follows: + - **uniform** - This approach distributes the datapoints for each bucket uniformly across the intersecting **ExplicitBounds**. The algorithm works as follows: - - If there are valid intersecting boundaries, the function evenly distributes the count across these boundaries. - - Calculate the count to be allocated to each boundary. - - If there is a remainder after dividing the count equally, it distributes the remainder by incrementing the count for some of the boundaries until the remainder is exhausted. + - If there are valid intersecting boundaries, the function evenly distributes the count across these boundaries. + - Calculate the count to be allocated to each boundary. + - If there is a remainder after dividing the count equally, it distributes the remainder by incrementing the count for some of the boundaries until the remainder is exhausted. _For example Given:_ 1. count = 10 2. Exponential Histogram Bounds: [10, 20] 3. Boundaries: [5, 10, 15, 20, 25] - 4. Intersecting Boundaries: [10, 15, 20] + 4. Intersecting Boundaries: [10, 15, 20] 5. Number of Intersecting Boundaries: 3 6. Using the formula: $count/numOfIntersections=10/3=3r1$ - _Uniform Allocation:_ - 7. Start with zeros: [0, 0, 0, 0, 0] 8. Allocate 3 to each: [0, 3, 3, 3, 0] 9. Distribute remainder $r$ 1: [0, 4, 3, 3, 0] 10. Final Counts: [0, 4, 3, 3, 0] - - __random__ - This approach distributes the datapoints for each bucket randomly across the intersecting __ExplicitBounds__. This approach works in a similar manner to the uniform distribution algorithm with the main difference being that points are distributed randomly instead of uniformly. This works as follows: + - **random** - This approach distributes the datapoints for each bucket randomly across the intersecting **ExplicitBounds**. This approach works in a similar manner to the uniform distribution algorithm with the main difference being that points are distributed randomly instead of uniformly. This works as follows: - If there are valid intersecting boundaries, calculate the proportion of the count that should be allocated to each boundary based on the overlap of the boundary with the provided range (lower to upper). - For each boundary, a random fraction of the calculated proportion is allocated. - Any remaining count (_due to rounding or random distribution_) is then distributed randomly among the intersecting boundaries. - If the bucket range does not intersect with any boundaries, the entire count is assigned to the start boundary. -- `ExplicitBounds` represents the list of bucket boundaries for the new histogram. This argument is __required__ and __cannot be empty__. +- `ExplicitBounds` represents the list of bucket boundaries for the new histogram. This argument is **required** and **cannot be empty**. -__WARNINGS:__ +**WARNINGS:** -- The process of converting an ExponentialHistogram to an Explicit Histogram is not perfect and may result in a loss of precision. It is important to define an appropriate set of bucket boundaries and identify the best distribution approach for your data in order to minimize this loss. +- The process of converting an ExponentialHistogram to an Explicit Histogram is not perfect and may result in a loss of precision. It is important to define an appropriate set of bucket boundaries and identify the best distribution approach for your data in order to minimize this loss. For example, selecting Boundaries that are too high or too low may result histogram buckets that are too wide or too narrow, respectively. -- __Negative Bucket Counts__ are not supported in Explicit Histograms, as such negative bucket counts are ignored. +- **Negative Bucket Counts** are not supported in Explicit Histograms, as such negative bucket counts are ignored. -- __ZeroCounts__ are only allocated if the ExplicitBounds array contains a zero boundary. That is, if the Explicit Boundaries that you provide does not start with `0`, the function will not allocate any zero counts from the Exponential Histogram. +- **ZeroCounts** are only allocated if the ExplicitBounds array contains a zero boundary. That is, if the Explicit Boundaries that you provide does not start with `0`, the function will not allocate any zero counts from the Exponential Histogram. This function should only be used when Exponential Histograms are not suitable for the downstream consumers or if upstream metric sources are unable to generate Explicit Histograms. -__Example__: +**Example**: - `convert_exponential_histogram_to_histogram("random", [0.0, 10.0, 100.0, 1000.0, 10000.0])` @@ -601,7 +595,7 @@ Examples: - `aggregate_on_attributes("max") where metric.name == "system.memory.usage"` - `aggregate_on_attributes("max", []) where metric.name == "system.memory.usage"` -The `aggregate_on_attributes` function can also be used in conjunction with +The `aggregate_on_attributes` function can also be used in conjunction with [keep_matching_keys](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs#keep_matching_keys) or [delete_matching_keys](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs#delete_matching_keys). @@ -645,7 +639,7 @@ Examples: - `aggregate_on_attribute_value("sum", "attr1", ["val1", "val2"], "new_val") where metric.name == "system.memory.usage"` -The `aggregate_on_attribute_value` function can also be used in conjunction with +The `aggregate_on_attribute_value` function can also be used in conjunction with [keep_matching_keys](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs#keep_matching_keys) or [delete_matching_keys](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs#delete_matching_keys). @@ -742,7 +736,6 @@ Sanitization examples: ``` * Span name after applying `set_semconv_span_name("1.40.0")`: `GET /api/v1/users/{id}` - Backward compatibility: `set_semconv_span_name` will map the following attributes to their equivalents per the v1.39.0 semantic conventions: | v1.40.0 Attribute | Older attribute | @@ -771,7 +764,7 @@ transform: trace_statements: # accessing a map with a key that does not exist will return nil. - set(span.attributes["test"], "pass") where span.attributes["test"] == nil -``` +``` ### Rename attribute There are 2 ways to rename an attribute key: @@ -784,7 +777,7 @@ transform: trace_statements: - set(resource.attributes["namespace"], resource.attributes["k8s.namespace.name"]) - delete_key(resource.attributes, "k8s.namespace.name") -``` +``` Or you can update the key using regex: @@ -793,7 +786,7 @@ transform: error_mode: ignore trace_statements: - replace_all_patterns(resource.attributes, "key", "k8s\\.namespace\\.name", "namespace") -``` +``` ### Move field to attribute Set attribute `body` to the value of the log body: @@ -803,10 +796,10 @@ transform: error_mode: ignore log_statements: - set(log.attributes["body"], log.body) -``` +``` ### Combine two attributes -Set attribute `test` to the value of attributes `"foo"` and `"bar"` combined. +Set attribute `test` to the value of attributes `"foo"` and `"bar"` combined. ```yaml transform: error_mode: ignore @@ -957,7 +950,6 @@ service: See [CONTRIBUTING.md](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/transformprocessor/CONTRIBUTING.md). - ## Warnings The Transform Processor uses the [OpenTelemetry Transformation Language](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/README.md) (OTTL) which allows users to modify all aspects of their telemetry. Some specific risks are listed below, but this is not an exhaustive list. In general, understand your data before using the Transform Processor. diff --git a/processor/unrollprocessor/README.md b/processor/unrollprocessor/README.md index 602bb98596f8f..e2bd5e9e4f8b0 100644 --- a/processor/unrollprocessor/README.md +++ b/processor/unrollprocessor/README.md @@ -4,7 +4,6 @@ The Unroll Processor takes log records with slice bodies and expands each element of the slice into its own log record. This allows for better processing and analysis of structured log data that contains arrays or lists. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | @@ -21,7 +20,6 @@ record. This allows for better processing and analysis of structured log data th - Logs - ## How it works The Unroll Processor processes log records through the following steps: @@ -51,8 +49,6 @@ unroll: recursive: false ``` - - ## Examples ### Basic Usage diff --git a/receiver/activedirectorydsreceiver/README.md b/receiver/activedirectorydsreceiver/README.md index beb3e98686439..aee45b367be46 100644 --- a/receiver/activedirectorydsreceiver/README.md +++ b/receiver/activedirectorydsreceiver/README.md @@ -4,7 +4,6 @@ The Active Directory Domain Services Receiver scrapes metrics relating to an Active Directory domain controller using the Windows Performance Counters. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/aerospikereceiver/README.md b/receiver/aerospikereceiver/README.md index 4a381eeecb07a..c4d5948421354 100644 --- a/receiver/aerospikereceiver/README.md +++ b/receiver/aerospikereceiver/README.md @@ -3,7 +3,6 @@ The Aerospike receiver is designed to collect performance metrics from one or more Aerospike nodes. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -21,7 +20,6 @@ connect and collect. Aerospike versions 4.9, 5.x, and 6.x are supported. - ## Configuration Configuration parameters: diff --git a/receiver/apachereceiver/README.md b/receiver/apachereceiver/README.md index 34ae54963d000..b4ffc2fbca068 100644 --- a/receiver/apachereceiver/README.md +++ b/receiver/apachereceiver/README.md @@ -3,7 +3,6 @@ The Apache Web Server Receiver fetches stats from an Apache Web Server instance using the `server-status?auto` endpoint. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -25,7 +24,6 @@ This receiver supports Apache Web Server version 2.4.13+. In order to receive server statistics, you must configure the server's `httpd.conf` file to [enable status support](https://httpd.apache.org/docs/2.4/mod/mod_status.html). - ### Configuration The following settings are required: diff --git a/receiver/apachesparkreceiver/README.md b/receiver/apachesparkreceiver/README.md index e9522e24b5df5..5464f058fafc8 100644 --- a/receiver/apachesparkreceiver/README.md +++ b/receiver/apachesparkreceiver/README.md @@ -5,7 +5,6 @@ The Apache Spark Receiver fetches metrics for an Apache Spark cluster through th the `/metrics/json`, `/api/v1/applications/[app-id]/stages`, `/api/v1/applications/[app-id]/executors`, and `/api/v1/applications/[app-id]/jobs` endpoints. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/receiver/awscloudwatchreceiver/README.md b/receiver/awscloudwatchreceiver/README.md index be018e944b7d8..dd3637b453e22 100644 --- a/receiver/awscloudwatchreceiver/README.md +++ b/receiver/awscloudwatchreceiver/README.md @@ -4,7 +4,6 @@ The AWS CloudWatch Receiver receives Cloudwatch events from [AWS CloudWatch](https://aws.amazon.com/cloudwatch/) via the [AWS SDK for Cloudwatch Logs](https://docs.aws.amazon.com/sdk-for-go/api/service/cloudwatchlogs/) - | Status | | | ------------- |-----------| | Stability | [alpha]: logs, metrics | diff --git a/receiver/awscontainerinsightreceiver/README.md b/receiver/awscontainerinsightreceiver/README.md index d2dce84ddf059..5d1e5b6c9bf6f 100644 --- a/receiver/awscontainerinsightreceiver/README.md +++ b/receiver/awscontainerinsightreceiver/README.md @@ -15,12 +15,12 @@ ## Overview -AWS Container Insights Receiver (`awscontainerinsight`) is an AWS specific receiver that supports [CloudWatch Container Insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/ContainerInsights.html). CloudWatch Container Insights collect, aggregate, -and summarize metrics and logs from your containerized applications and microservices. Data are collected as as performance log events +AWS Container Insights Receiver (`awscontainerinsight`) is an AWS specific receiver that supports [CloudWatch Container Insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/ContainerInsights.html). CloudWatch Container Insights collect, aggregate, +and summarize metrics and logs from your containerized applications and microservices. Data are collected as as performance log events using [embedded metric format](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Embedded_Metric_Format.html). From the EMF data, Amazon CloudWatch can create the aggregated CloudWatch metrics at the cluster, node, pod, task, and service level. CloudWatch Container Insights has been supported by [ECS Agent](https://github.com/aws/amazon-ecs-agent) and [CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent) to collect infrastructure metrics for many resources such as such as CPU, memory, disk, and network. To migrate existing customers to use OpenTelemetry, AWS Container Insights Receiver (together with CloudWatch EMF Exporter) aims to support the same CloudWatch Container Insights experience for the following platforms: - * Amazon ECS + * Amazon ECS * Amazon EKS * Kubernetes platforms on Amazon EC2 @@ -40,7 +40,7 @@ receivers: prefer_full_pod_name: false add_full_pod_name_metric_label: false ``` -There is no need to provide any parameters since they are all optional. +There is no need to provide any parameters since they are all optional. **collection_interval (optional)** @@ -62,7 +62,7 @@ The "PodName" attribute is set based on the name of the relevant controllers lik The "FullPodName" attribute is the pod name including suffix. If false FullPodName label is not added. The default value is false -## Sample configuration for Container Insights +## Sample configuration for Container Insights This is a sample configuration for AWS Container Insights using the `awscontainerinsight` and `awsemfexporter` for an EKS cluster: ``` # create namespace @@ -335,7 +335,7 @@ kubectl apply -f config.yaml | cluster_failed_node_count | Count | | cluster_node_count | Count | -

+

| Resource Attribute | |--------------------| | ClusterName | @@ -344,15 +344,15 @@ kubectl apply -f config.yaml | Version | | Sources | -

-

+

+

### Cluster Namespace | Metric | Unit | |----------------------------------|-------| | namespace_number_of_running_pods | Count | -

+

| Resource Attribute | |--------------------| | ClusterName | @@ -363,16 +363,15 @@ kubectl apply -f config.yaml | Sources | | kubernete | -

-

+

+

### Cluster Service | Metric | Unit | |--------------------------------|-------| | service_number_of_running_pods | Count | - -

+

| Resource Attribute | |--------------------| | ClusterName | @@ -384,9 +383,8 @@ kubectl apply -f config.yaml | Sources | | kubernete | - -

-

+

+

### Node | Metric | Unit | @@ -426,7 +424,7 @@ kubectl apply -f config.yaml | node_number_of_running_containers | Count | | node_number_of_running_pods | Count | -

+

| Resource Attribute | |----------------------| | ClusterName | @@ -437,8 +435,8 @@ kubectl apply -f config.yaml | Sources | | kubernete | -

-

+

+

### Node Disk IO | Metric | Unit | @@ -454,7 +452,7 @@ kubectl apply -f config.yaml | node_diskio_io_service_bytes_total | Bytes/Second | | node_diskio_io_service_bytes_write | Bytes/Second | -

+

| Resource Attribute | |----------------------| | AutoScalingGroupName | @@ -468,8 +466,9 @@ kubectl apply -f config.yaml | Version | | Sources | | kubernete | -

-

+ +

+

### Node Filesystem | Metric | Unit | @@ -481,7 +480,7 @@ kubectl apply -f config.yaml | node_filesystem_usage | Bytes | | node_filesystem_utilization | Percent | -

+

| Resource Attribute | |----------------------| | AutoScalingGroupName | @@ -496,8 +495,9 @@ kubectl apply -f config.yaml | Version | | Sources | | kubernete | -

-

+ +

+

### Node Network | Metric | Unit | @@ -512,7 +512,7 @@ kubectl apply -f config.yaml | node_interface_network_tx_errors | Count/Second | | node_interface_network_tx_packets | Count/Second | -

+

| Resource Attribute | |----------------------| | AutoScalingGroupName | @@ -525,8 +525,9 @@ kubectl apply -f config.yaml | interface | | Sources | | kubernete | -

-

+ +

+

### Pod | Metric | Unit | @@ -565,8 +566,8 @@ kubectl apply -f config.yaml | pod_network_tx_dropped | Count/Second | | pod_network_tx_errors | Count/Second | | pod_network_tx_packets | Count/Second | -| pod_number_of_container_restarts | Count | -| pod_number_of_containers | Count | +| pod_number_of_container_restarts | Count | +| pod_number_of_containers | Count | | pod_number_of_running_containers | Count | | Resource Attribute | @@ -585,7 +586,7 @@ kubectl apply -f config.yaml | kubernete | | pod_status | -

+

### Pod Network | Metric | Unit | @@ -600,7 +601,7 @@ kubectl apply -f config.yaml | pod_interface_network_tx_errors | Count/Second | | pod_interface_network_tx_packets | Count/Second | -

+

| Resource Attribute | |----------------------| | AutoScalingGroupName | @@ -617,9 +618,9 @@ kubectl apply -f config.yaml | Sources | | kubernete | | pod_status | -

-

+

+

### Container | Metric | Unit | @@ -647,7 +648,7 @@ kubectl apply -f config.yaml | container_memory_working_set | Bytes | | number_of_container_restarts | Count | -

+

| Resource Attribute | |-----------------------------------| @@ -667,7 +668,7 @@ kubectl apply -f config.yaml | kubernetes | | container_status | | container_status_reason | -| container_last_termination_reason | +| container_last_termination_reason | The attribute `container_status_reason` is present only when `container_status` is in "Waiting" or "Terminated" State. The attribute `container_last_termination_reason` is present only when `container_status` is in "Terminated" State. @@ -758,6 +759,7 @@ To deploy to an ECS cluster check this [doc](https://aws-otel.github.io/docs/set | instance_network_tx_errors | Count/Second | | instance_network_tx_packets | Count/Second | | instance_number_of_running_tasks | Count | +

| Resource Attribute | @@ -827,6 +829,7 @@ To deploy to an ECS cluster check this [doc](https://aws-otel.github.io/docs/set | ContainerInstanceId | | InstanceId | | EBSVolumeId | +



@@ -855,6 +858,7 @@ To deploy to an ECS cluster check this [doc](https://aws-otel.github.io/docs/set | ContainerInstanceId | | InstanceId | | EBSVolumeId | +



diff --git a/receiver/awscontainerinsightreceiver/design.md b/receiver/awscontainerinsightreceiver/design.md index 74260407c7675..29e3b9fac7188 100644 --- a/receiver/awscontainerinsightreceiver/design.md +++ b/receiver/awscontainerinsightreceiver/design.md @@ -1,21 +1,20 @@ # Design of AWS Container Insights Receiver - ## Container Insights Architecture for EKS ![architecture](images/eks-design.png) ## Container Insights Architecture for ECS ![architecture](images/ecs-design.png) -## Components of AWS Container Insights Receiver +## Components of AWS Container Insights Receiver `awscontainerinsight` collects data from two main sources: -* `cadvisor` - * A customized `cadvisor` lib is embedded inside the receiver. The `cadvisor` setting is tweaked for Container Insights use cases. For example, only certain metrics are collected and only certain `cgroup` is included. - * The receiver generates Container Insights specific metrics from the raw metrics provided by `cadvisor`. The metrics are categorized as different infrastructure layers like node, node filesystem, node disk io, node network, pod, pod network, container, and container filesystem. - * Some pod/container related labels like podName, podId, namespace, containerName are extracted from the container spec provided by `cadvisor`. This labels will be added as resource attributes for the metrics and the AWS Container Insights processor needs those attributes to do further processing of the metrics. +* `cadvisor` + * A customized `cadvisor` lib is embedded inside the receiver. The `cadvisor` setting is tweaked for Container Insights use cases. For example, only certain metrics are collected and only certain `cgroup` is included. + * The receiver generates Container Insights specific metrics from the raw metrics provided by `cadvisor`. The metrics are categorized as different infrastructure layers like node, node filesystem, node disk io, node network, pod, pod network, container, and container filesystem. + * Some pod/container related labels like podName, podId, namespace, containerName are extracted from the container spec provided by `cadvisor`. This labels will be added as resource attributes for the metrics and the AWS Container Insights processor needs those attributes to do further processing of the metrics. * `k8sapiserver` - * Collects cluster-level metrics from k8s api server + * Collects cluster-level metrics from k8s api server * The receiver is designed to run as daemonset. This guarantees that only one receiver is running per cluster node. To make sure cluster-level metrics are not duplicated, the receiver integrate with K8s client which support leader election API. It leverages k8s configmap resource as some sort of LOCK primitive. The deployment will create a dedicate configmap as the lock resource. If one receiver is required to elect a leader, it will try to lock (via Create/Update) the configmap. The API will ensure one of the receivers hold the lock to be the leader. The leader continually “heartbeats” to claim its leaderships, and the other candidates periodically make new attempts to become the leader. This ensures that a new leader will be elected quickly, if the current leader fails for some reason. The following two packages are used to decorate metrics: @@ -24,7 +23,6 @@ The following two packages are used to decorate metrics: * get cpu and mem capacity of the host from `gopsutil` * retrieve info about ebs volume, autoscaling group and cluster name from ec2 metadata and ec2 apis - * `stores` * pod store: * access Kubelet to list all Pod objects running on the same node, and cache all the Pod objects in memory @@ -36,7 +34,7 @@ The following two packages are used to decorate metrics: 4. Container ID and Status 5. Pod Status 6. Pod Labels - * service store: + * service store: * add the corresponding attribute `Service` to the metrics for the relevant pods * call "List & Watch" for "Endpoint" objects from Kubernetes API server. In this mode, when there is any change to “Endpoint”, agent will get the notification of the change. The benefit is that the agent could get Endpoint information update in time. If the "Endpoint" resources just occasionally slightly changes, the impact to API server is supposed to be minimal. diff --git a/receiver/awsecscontainermetricsreceiver/README.md b/receiver/awsecscontainermetricsreceiver/README.md index 1fa8136f2a998..d54ba77696bcb 100644 --- a/receiver/awsecscontainermetricsreceiver/README.md +++ b/receiver/awsecscontainermetricsreceiver/README.md @@ -18,7 +18,6 @@ AWS ECS Container Metrics Receiver (`awsecscontainermetrics`) reads task metadat This receiver works only for [ECS Task Metadata Endpoint V4](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html). Amazon ECS tasks on Fargate that use platform version 1.4.0 or later and Amazon ECS tasks on Amazon EC2 that are running at least version 1.39.0 of the Amazon ECS container agent can utilize this receiver. For more information, see [Amazon ECS Container Agent Versions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-agent-versions.html). - ## Configuration Example: @@ -35,7 +34,6 @@ This receiver collects task metadata and container stats at a fixed interval and default: `20s` - ## Enabling the AWS ECS Container Metrics Receiver To enable the awsecscontainermetrics receiver, add the name under receiver section in the OpenTelemetry config file. By default, the receiver scrapes the ECS task metadata endpoint every 20s and collects all metrics (For the full list of metrics, see [Available Metrics](#available-metrics)). @@ -115,66 +113,65 @@ service: ## Available Metrics Following is the full list of metrics emitted by this receiver. -Task Level Metrics | Container Level Metrics | Unit ------------- | ------------- | -------------------- -ecs.task.memory.usage | container.memory.usage | Bytes -ecs.task.memory.usage.max | container.memory.usage.max | Bytes -ecs.task.memory.usage.limit | container.memory.usage.limit | Bytes -ecs.task.memory.reserved | container.memory.reserved | MiB -ecs.task.memory.utilized | container.memory.utilized | MiB -ecs.task.cpu.usage.total | container.cpu.usage.total | Nanoseconds -ecs.task.cpu.usage.kernelmode | container.cpu.usage.kernelmode | Nanoseconds -ecs.task.cpu.usage.usermode | container.cpu.usage.usermode | Nanoseconds -ecs.task.cpu.usage.system | container.cpu.usage.system | Nanoseconds -ecs.task.cpu.usage.vcpu | container.cpu.usage.vcpu | vCPU -ecs.task.cpu.cores | container.cpu.cores | Count -ecs.task.cpu.onlines | container.cpu.onlines | Count -ecs.task.cpu.reserved | container.cpu.reserved | vCPU -ecs.task.cpu.utilized | container.cpu.utilized | Percent -ecs.task.network.rate.rx | container.network.rate.rx | Bytes/Second -ecs.task.network.rate.tx | container.network.rate.tx | Bytes/Second -ecs.task.network.io.usage.rx_bytes | container.network.io.usage.rx_bytes | Bytes -ecs.task.network.io.usage.rx_packets | container.network.io.usage.rx_packets | Count -ecs.task.network.io.usage.rx_errors | container.network.io.usage.rx_errors | Count -ecs.task.network.io.usage.rx_dropped | container.network.io.usage.rx_dropped | Count -ecs.task.network.io.usage.tx_bytes | container.network.io.usage.tx_bytes | Bytes -ecs.task.network.io.usage.tx_packets | container.network.io.usage.tx_packets | Count -ecs.task.network.io.usage.tx_errors | container.network.io.usage.tx_errors | Count -ecs.task.network.io.usage.tx_dropped | container.network.io.usage.tx_dropped | Count -ecs.task.storage.read_bytes | container.storage.read_bytes| Bytes -ecs.task.storage.write_bytes | container.storage.write_bytes | Bytes -ecs.task.ephemeral_storage.utilized | | MiB -ecs.task.ephemeral_storage.reserved | | MiB - +| Task Level Metrics | Container Level Metrics | Unit | +|------------|-------------|---------------------| +| ecs.task.memory.usage | container.memory.usage | Bytes | +| ecs.task.memory.usage.max | container.memory.usage.max | Bytes | +| ecs.task.memory.usage.limit | container.memory.usage.limit | Bytes | +| ecs.task.memory.reserved | container.memory.reserved | MiB | +| ecs.task.memory.utilized | container.memory.utilized | MiB | +| ecs.task.cpu.usage.total | container.cpu.usage.total | Nanoseconds | +| ecs.task.cpu.usage.kernelmode | container.cpu.usage.kernelmode | Nanoseconds | +| ecs.task.cpu.usage.usermode | container.cpu.usage.usermode | Nanoseconds | +| ecs.task.cpu.usage.system | container.cpu.usage.system | Nanoseconds | +| ecs.task.cpu.usage.vcpu | container.cpu.usage.vcpu | vCPU | +| ecs.task.cpu.cores | container.cpu.cores | Count | +| ecs.task.cpu.onlines | container.cpu.onlines | Count | +| ecs.task.cpu.reserved | container.cpu.reserved | vCPU | +| ecs.task.cpu.utilized | container.cpu.utilized | Percent | +| ecs.task.network.rate.rx | container.network.rate.rx | Bytes/Second | +| ecs.task.network.rate.tx | container.network.rate.tx | Bytes/Second | +| ecs.task.network.io.usage.rx_bytes | container.network.io.usage.rx_bytes | Bytes | +| ecs.task.network.io.usage.rx_packets | container.network.io.usage.rx_packets | Count | +| ecs.task.network.io.usage.rx_errors | container.network.io.usage.rx_errors | Count | +| ecs.task.network.io.usage.rx_dropped | container.network.io.usage.rx_dropped | Count | +| ecs.task.network.io.usage.tx_bytes | container.network.io.usage.tx_bytes | Bytes | +| ecs.task.network.io.usage.tx_packets | container.network.io.usage.tx_packets | Count | +| ecs.task.network.io.usage.tx_errors | container.network.io.usage.tx_errors | Count | +| ecs.task.network.io.usage.tx_dropped | container.network.io.usage.tx_dropped | Count | +| ecs.task.storage.read_bytes | container.storage.read_bytes | Bytes | +| ecs.task.storage.write_bytes | container.storage.write_bytes | Bytes | +| ecs.task.ephemeral_storage.utilized | | MiB | +| ecs.task.ephemeral_storage.reserved | | MiB | ## Resource Attributes and Metrics Labels Metrics emitted by this receiver comes with a set of resource attributes. These resource attributes can be converted to metrics labels using appropriate processors/exporters (See `Full Configuration Examples` section below). Finally, these metrics labels can be set as metrics dimensions while exporting to desired destinations. Check the following table to see available resource attributes for Task and Container level metrics. Container level metrics have three additional attributes than task level metrics. -Resource Attributes for Task Level Metrics | Resource Attributes for Container Level Metrics --------------------- | ----------------------------- -aws.ecs.cluster.name | aws.ecs.cluster.name -aws.ecs.task.family | aws.ecs.task.family -aws.ecs.task.arn | aws.ecs.task.arn -aws.ecs.task.id | aws.ecs.task.id -aws.ecs.task.revision | aws.ecs.task.revision -aws.ecs.service.name | aws.ecs.service.name -cloud.availability_zone | cloud.availability_zone -cloud.account.id | cloud.account.id -cloud.region | cloud.region -aws.ecs.task.pull_started_at | aws.ecs.container.started_at -aws.ecs.task.pull_stopped_at | aws.ecs.container.finished_at -aws.ecs.task.known_status | aws.ecs.container.know_status -aws.ecs.launch_type | aws.ecs.launch_type -  | aws.ecs.container.created_at -  | container.name -  | container.id -  | aws.ecs.docker.name -  | container.image.tag -  | aws.ecs.container.image.id -  | aws.ecs.container.exit_code +| Resource Attributes for Task Level Metrics | Resource Attributes for Container Level Metrics | +|--------------------|------------------------------| +| aws.ecs.cluster.name | aws.ecs.cluster.name | +| aws.ecs.task.family | aws.ecs.task.family | +| aws.ecs.task.arn | aws.ecs.task.arn | +| aws.ecs.task.id | aws.ecs.task.id | +| aws.ecs.task.revision | aws.ecs.task.revision | +| aws.ecs.service.name | aws.ecs.service.name | +| cloud.availability_zone | cloud.availability_zone | +| cloud.account.id | cloud.account.id | +| cloud.region | cloud.region | +| aws.ecs.task.pull_started_at | aws.ecs.container.started_at | +| aws.ecs.task.pull_stopped_at | aws.ecs.container.finished_at | +| aws.ecs.task.known_status | aws.ecs.container.know_status | +| aws.ecs.launch_type | aws.ecs.launch_type | +|   | aws.ecs.container.created_at | +|   | container.name | +|   | container.id | +|   | aws.ecs.docker.name | +|   | container.image.tag | +|   | aws.ecs.container.image.id | +|   | aws.ecs.container.exit_code | ## Full Configuration Examples -This receiver emits 54 unique metrics. Customer may not want to send all of them to destinations. Following sections will show full configuration files for filtering and transforming existing metrics with different processors/exporters. +This receiver emits 54 unique metrics. Customer may not want to send all of them to destinations. Following sections will show full configuration files for filtering and transforming existing metrics with different processors/exporters. ### 1. Full configuration for task level metrics The following example shows a full configuration to get most useful task level metrics. It uses `awsecscontainermetrics` receiver to collect all the resource usage metrics from ECS task metadata endpoint. It applies `filter` processor to select only 8 task-level metrics and update metric names using `metricstransform` processor. It also renames the resource attributes using `resource` processor which will be used as metric dimensions in the Amazon CloudWatch `awsemf` exporter. Finally, it sends the metrics to CloudWatch using `awsemf` exporter under the `/aws/ecs/containerinsights/{ClusterName}/performance` namespace where the `{ClusterName}` placeholder will be replaced with actual cluster name. Check the [AWS EMF Exporter](https://aws-otel.github.io/docs/getting-started/cloudwatch-metrics) documentation to see and explore the metrics in Amazon CloudWatch. @@ -266,7 +263,6 @@ service: exporters: [ awsemf ] ``` - ### 2. Full configuration for task- and container-level metrics The following example shows a full configuration to get most useful task- and container-level metrics. It uses `awsecscontainermetrics` receiver to collect all the resource usage metrics from ECS task metadata endpoint. It applies `filter` processor to select only 8 task- and container-level metrics and update metric names using `metricstransform` processor. It also renames the resource attributes using `resource` processor which will be used as metric dimensions in the Amazon CloudWatch `awsemf` exporter. Finally, it sends the metrics to CloudWatch using `awsemf` exporter under the /`aws/ecs/containerinsights/{ClusterName}/performance` namespace where the `{ClusterName}` placeholder will be replaced with actual cluster name. Check the [AWS EMF Exporter](https://aws-otel.github.io/docs/getting-started/cloudwatch-metrics) documentation to see and explore the metrics in Amazon CloudWatch. @@ -375,4 +371,3 @@ service: ## Reference 1. [Setup OpenTelemetry Collector on Amazon ECS](https://aws-otel.github.io/docs/setup/ecs) 2. [Getting Started with ECS Container Metrics Receiver in the OpenTelemetry Collector](https://aws-otel.github.io/docs/components/ecs-metrics-receiver) - diff --git a/receiver/awsfirehosereceiver/README.md b/receiver/awsfirehosereceiver/README.md index d886449e7d65f..fffefef7f8fba 100644 --- a/receiver/awsfirehosereceiver/README.md +++ b/receiver/awsfirehosereceiver/README.md @@ -4,7 +4,6 @@ The AWS Kinesis Data Firehose Receiver is for ingesting delivery stream messages and parsing the records received based on the configured record type. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics, logs | diff --git a/receiver/awslambdareceiver/README.md b/receiver/awslambdareceiver/README.md index fc8fbdb77462d..cfbfb36bdd719 100644 --- a/receiver/awslambdareceiver/README.md +++ b/receiver/awslambdareceiver/README.md @@ -14,8 +14,8 @@ ## Overview -A receiver for collecting logs & metrics from AWS services via Lambda invocations. -AWS Lambda is a popular serverless service used extensively for event-driven architectures. +A receiver for collecting logs & metrics from AWS services via Lambda invocations. +AWS Lambda is a popular serverless service used extensively for event-driven architectures. Many AWS services (S3, CloudWatch, SNS, SQS) can trigger Lambda functions, making it an ideal entry point for collecting data from AWS services. This receiver is designed to run as part of an OpenTelemetry Collector deployed as an AWS Lambda function. @@ -213,8 +213,8 @@ The following receiver configuration parameters are supported. | Name | Description | |:-----------------------|:--------------------------------------------------------| -| `s3::encoding` | Optional encoder to use for S3 event processing | -| `cloudwatch::encoding` | Optional encoder to use for CloudWatch event processing | +| `s3::encoding` | Optional encoder to use for S3 event processing | +| `cloudwatch::encoding` | Optional encoder to use for CloudWatch event processing | Consider following notes on default behaviors: @@ -380,8 +380,8 @@ service: exporters: [otlp_http] ``` -For this deployment configuration, when receiver is triggered by a CloudWatch Logs subscription filter, the CloudWatch -messages will be extracted and converted to an OpenTelemetry log record. +For this deployment configuration, when receiver is triggered by a CloudWatch Logs subscription filter, the CloudWatch +messages will be extracted and converted to an OpenTelemetry log record. These logs then get forwarded to an OTLP listener via the `otlp_http` exporter. ### Example 4: Arbitrary S3 content (logs or metrics) @@ -401,7 +401,7 @@ service: exporters: [otlp_http] ``` -For this deployment configuration, when receiver is triggered by an S3 event, +For this deployment configuration, when receiver is triggered by an S3 event, - Logs: Content of the S3 object will be added to an OpenTelemetry log record. If content is string, then it will be added as-is. - Metrics: Metrics will be decoded using `awscloudwatchmetricstreams_encoding` extension. @@ -428,10 +428,10 @@ The Lambda function requires the following IAM permissions: ## Error Handling - Detailed error information is logged for troubleshooting - + These logs can be views via the configured CloudWatch Logs group for the Lambda function. -- Error retrying +- Error retrying Error retrying can be configured through the Lambda deployment setting. Read more about at [AWS error handling for asynchronous invocations](https://docs.aws.amazon.com/lambda/latest/dg/invocation-async-configuring.html). @@ -496,7 +496,7 @@ Check the CloudWatch logs for detailed information. > [!NOTE] > Using AWS CLI, you can use `--timeout` option to increase currently configured Lambda timeout for custom invocations. -> Also note that errors resulting from this manual trigger are not retained back to S3 failure destination. +> Also note that errors resulting from this manual trigger are not retained back to S3 failure destination. > This is because Lambda only retains errors for asynchronous invocations. To perform a dry run, use the following command with `dryrun` set to `true`: diff --git a/receiver/awss3receiver/README.md b/receiver/awss3receiver/README.md index 942738da696b4..2019f21508e4b 100644 --- a/receiver/awss3receiver/README.md +++ b/receiver/awss3receiver/README.md @@ -85,7 +85,6 @@ By default, the receiver understands the following encodings: The `encodings` options allows you to specify Encoding Extensions to use to decode keys with matching suffixes. - ### Example Configuration ```yaml diff --git a/receiver/azureblobreceiver/README.md b/receiver/azureblobreceiver/README.md index 3e615489801a9..8af07a213ff42 100644 --- a/receiver/azureblobreceiver/README.md +++ b/receiver/azureblobreceiver/README.md @@ -12,7 +12,6 @@ [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib - This receiver reads logs and trace data from [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs/). ## Modes of Operation @@ -50,8 +49,6 @@ Authenticating using service principal requires configuration of the following a - `client_secret` - `storage_account_url:` Azure Storage Account url - - The service principal method also requires the [Storage Blob Data Contributor](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/storage#storage-blob-data-contributor) role on the logs and traces containers. ### Example configurations @@ -94,4 +91,3 @@ receivers: In **Event Hub mode**, the receiver subscribes to [blob events](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-event-overview) published by Azure Blob Storage and handled by Azure Event Hub. When it receives a `Blob Create` event, it reads the logs or traces from the corresponding blob and deletes it after processing. In **polling mode**, the receiver periodically lists all blobs in the configured containers, processes each blob, and deletes it after processing. - diff --git a/receiver/azureeventhubreceiver/README.md b/receiver/azureeventhubreceiver/README.md index 670d837d040b0..88e2ff6487841 100644 --- a/receiver/azureeventhubreceiver/README.md +++ b/receiver/azureeventhubreceiver/README.md @@ -209,7 +209,6 @@ attributes. The table below summarizes the mapping between the [Azure common log format](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/resource-logs-schema) and the OpenTelemetry attributes. - | Azure | OpenTelemetry | |----------------------------------|----------------------------------------| | callerIpAddress (optional) | network.peer.address (attribute) | diff --git a/receiver/azuremonitorreceiver/README.md b/receiver/azuremonitorreceiver/README.md index 68c18dc0064d8..3d6cb09fb687e 100644 --- a/receiver/azuremonitorreceiver/README.md +++ b/receiver/azuremonitorreceiver/README.md @@ -61,7 +61,7 @@ Authenticating using managed identities has the following optional settings: ### Filtering metrics The `metrics` configuration setting is designed to **limit** scraping to specific metrics and their particular aggregations. -It accepts a nested map where +It accepts a nested map where - the key of the top-level is the Azure Metric Namespace, - the key of the nested map is an Azure Metric Name, - and the map values are a list of aggregation methods (e.g., Average, Minimum, Maximum, Total, Count). @@ -73,7 +73,7 @@ It accepts a nested map where > - **Case Insensitive**: The letter case of the Namespaces, Metric names, and Aggregations does not affect the functionality. > [!WARNING] -> If you started providing a `metrics` configuration for a namespace, you have to specify all the metrics and their +> If you started providing a `metrics` configuration for a namespace, you have to specify all the metrics and their > aggregations for that namespace. Otherwise, these metrics will be ignored. Scraping limited metrics and aggregations: @@ -217,7 +217,7 @@ Details about the metrics scraped by this receiver can be found in [Supported me ## Azure API calls summary At each collection interval, here are the different Azure API that can be called. -It can be useful to know that, in order to configure the client permission in Azure or to choose the +It can be useful to know that, in order to configure the client permission in Azure or to choose the right configuration based on your needs. ### [Subscriptions - Get](https://learn.microsoft.com/en-us/rest/api/resources/subscriptions/get?view=rest-resources-2022-12-01&tabs=HTTP) @@ -266,14 +266,14 @@ cardinality: once per res type and **composite key > *page size has not been clearly identified, reading the documentation. Even Chat Bots lose themselves > with the "top"/"$top" filter that doesn't seem related, and give random results from 10 to 1000... -> +> > **the composite key is an identifier formed with info retrieved in metric definitions. Useful to group and reduce the number of metrics calls. -It is composed by +It is composed by > - dimensions, -> - aggregations, +> - aggregations, > - minimum timegrain. -> +> > It is used to get several metrics in one request. ## Troubleshooting diff --git a/receiver/azuremonitorreceiver/concurrency_bench_report.md b/receiver/azuremonitorreceiver/concurrency_bench_report.md index 5a1915d997555..b73009a9e086c 100644 --- a/receiver/azuremonitorreceiver/concurrency_bench_report.md +++ b/receiver/azuremonitorreceiver/concurrency_bench_report.md @@ -34,4 +34,3 @@ BenchmarkMutexMapImplLarge-8 748.2 ns/op 31 B/op 1 allocs/op ## Conclusion The generic concurrent-map implementation offers the best performance and scalability for concurrent workloads in Go. The classic mutex-protected map is a good fallback for simpler cases. Avoid sync.Map for intensive workloads. - diff --git a/receiver/azuremonitorreceiver/deepcopy_bench_report.md b/receiver/azuremonitorreceiver/deepcopy_bench_report.md index e6b98094d2e04..56fce8f96f1ff 100644 --- a/receiver/azuremonitorreceiver/deepcopy_bench_report.md +++ b/receiver/azuremonitorreceiver/deepcopy_bench_report.md @@ -18,59 +18,59 @@ This benchmark compares several Go libraries for deep copying the armresources.G ```go import ( - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/brunoga/deep" - "github.com/huandu/go-clone" - "github.com/mohae/deepcopy" - "github.com/mitchellh/copystructure" - "github.com/barkimedes/go-deepcopy" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/brunoga/deep" + "github.com/huandu/go-clone" + "github.com/mohae/deepcopy" + "github.com/mitchellh/copystructure" + "github.com/barkimedes/go-deepcopy" ) func BenchmarkDeepCopyLibraries(b *testing.B) { - orig := armresources.GenericResourceExpanded{ - ID: to.Ptr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/myResource"), - Type: to.Ptr("Microsoft.Storage/storageAccounts"), - Name: to.Ptr("myResource"), - Location: to.Ptr("westeurope"), - Tags: map[string]*string{ - "env": to.Ptr("prod"), - }, - } - - b.Run("brunoga/deep", func(b *testing.B) { - for i := 0; i < b.N; i++ { - clonedAny, _ := deep.Copy(orig) - _ = clonedAny - } - }) - - b.Run("mohae/deepcopy", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = deepcopy.Copy(orig) - } - }) - - b.Run("barkimedes/go-deepcopy", func(b *testing.B) { - b.Skip("barkimedes/go-deepcopy does not support armresources.GenericResourceExpanded (panic reflect.Value.Set)") - for i := 0; i < b.N; i++ { - _, _ = barkdeep.Anything(orig) - } - }) - - b.Run("mitchellh/copystructure", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = copystructure.Copy(orig) - } - }) - - b.Run("huandu/go-clone", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = clone.Clone(orig).(armresources.GenericResourceExpanded) - } - }) + orig := armresources.GenericResourceExpanded{ + ID: to.Ptr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/myResource"), + Type: to.Ptr("Microsoft.Storage/storageAccounts"), + Name: to.Ptr("myResource"), + Location: to.Ptr("westeurope"), + Tags: map[string]*string{ + "env": to.Ptr("prod"), + }, + } + + b.Run("brunoga/deep", func(b *testing.B) { + for i := 0; i < b.N; i++ { + clonedAny, _ := deep.Copy(orig) + _ = clonedAny + } + }) + + b.Run("mohae/deepcopy", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = deepcopy.Copy(orig) + } + }) + + b.Run("barkimedes/go-deepcopy", func(b *testing.B) { + b.Skip("barkimedes/go-deepcopy does not support armresources.GenericResourceExpanded (panic reflect.Value.Set)") + for i := 0; i < b.N; i++ { + _, _ = barkdeep.Anything(orig) + } + }) + + b.Run("mitchellh/copystructure", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = copystructure.Copy(orig) + } + }) + + b.Run("huandu/go-clone", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = clone.Clone(orig).(armresources.GenericResourceExpanded) + } + }) } ``` diff --git a/receiver/carbonreceiver/README.md b/receiver/carbonreceiver/README.md index d57d745d351a2..9b8df45bee4c9 100644 --- a/receiver/carbonreceiver/README.md +++ b/receiver/carbonreceiver/README.md @@ -4,7 +4,6 @@ The [Carbon](https://github.com/graphite-project/carbon) Receiver supports Carbon's [plaintext protocol](https://graphite.readthedocs.io/en/stable/feeding-carbon.html#the-plaintext-protocol). - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -68,4 +67,3 @@ receivers: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/ciscoosreceiver/README.md b/receiver/ciscoosreceiver/README.md index 6db2176b30692..b366384a0c8a3 100644 --- a/receiver/ciscoosreceiver/README.md +++ b/receiver/ciscoosreceiver/README.md @@ -5,7 +5,6 @@ The Cisco OS Receiver is a modular receiver that collects metrics from Cisco net supports multiple scrapers for comprehensive network monitoring and provides native OpenTelemetry OTLP metrics generation. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/receiver/cloudflarereceiver/README.md b/receiver/cloudflarereceiver/README.md index 35c33ad89e7af..3e82ac7afa5bf 100644 --- a/receiver/cloudflarereceiver/README.md +++ b/receiver/cloudflarereceiver/README.md @@ -4,7 +4,6 @@ The Cloudflare Receiver allows Cloudflare's [LogPush Jobs](https://developers.cloudflare.com/logs/logpush/) to send logs over HTTPS from the Cloudflare logs aggregation system to an OpenTelemetry collector. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | @@ -64,7 +63,6 @@ If the receiver will be handling TLS termination: - The separator used to join nested fields in the log message when setting attributes. For example, if the log message contains a field `"RequestHeaders": { "Content-Type": "application/json" }`, and the `separator` is set to `.`, the attribute will be set as `RequestHeaders.Content_Type`. If the separator is set to `_`, it will be set as `RequestHeaders_Content_Type`. - `max_request_body_size`: configures the maximum allowed body size in bytes for a single request. Default: `20971520` (20MiB) - ### Example: ```yaml diff --git a/receiver/cloudfoundryreceiver/README.md b/receiver/cloudfoundryreceiver/README.md index 7bf1132b35242..2f6077d101c61 100644 --- a/receiver/cloudfoundryreceiver/README.md +++ b/receiver/cloudfoundryreceiver/README.md @@ -4,7 +4,6 @@ The Cloud Foundry Receiver connects to the RLP (Reverse Log Proxy) Gateway of the Cloud Foundry installation, typically available at the URL `https://log-stream.`. - | Status | | | ------------- |-----------| | Stability | [development]: logs | @@ -108,7 +107,7 @@ origin name is prepended to the metric name with `.` separator. All metrics eith ### Attributes The receiver maps the envelope attribute to the following OpenTelemetry attributes: - + * `source_id` - for applications, the GUID of the application, otherwise equal to `origin` For metrics originating with `rep` origin name (specific to applications), the following attributes are present: @@ -119,7 +118,6 @@ For metrics originating with `rep` origin name (specific to applications), the f * `process_type` - process type. Each application has exactly one process of type `web`, but many have any number of other processes - ## Logs The receiver maps loggregator envelopes of these types to the following OpenTelemetry log severity text and severity number: @@ -133,7 +131,7 @@ The receiver maps the envelope attribute tags to the following OpenTelemetry att * `source_id` - for applications, the GUID of the application, otherwise the GUID of the log generator * `source_type` - The source of the log, any subset of `{API|APP|CELL|HEALTH|LGR|RTR|SSH|STG}`, for `APP` type extra labels are separated by a dash, example: `APP/PROC/WEB` -* `instance_id` - numerical index of the origin. If origin is `rep` (`source_type` is `APP`) this is the application index. However, for other cases this is the instance index. +* `instance_id` - numerical index of the origin. If origin is `rep` (`source_type` is `APP`) this is the application index. However, for other cases this is the instance index. * `process_id` - process ID (GUID) * `process_instance_id` - unique ID of a process instance, should be treated as an opaque string * `process_type` - process type. Each application has exactly one process of type `web`, but many have any number of other processes diff --git a/receiver/collectdreceiver/README.md b/receiver/collectdreceiver/README.md index 56e5c817502e8..9b3c09e61d658 100644 --- a/receiver/collectdreceiver/README.md +++ b/receiver/collectdreceiver/README.md @@ -4,7 +4,6 @@ This receiver can receive data exported by the CollectD's `write_http` plugin. Only JSON format is supported. Authentication is not supported at this time. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -52,4 +51,3 @@ receivers: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/couchdbreceiver/README.md b/receiver/couchdbreceiver/README.md index b9d53f9f2206d..d0f06bd0cf7f0 100644 --- a/receiver/couchdbreceiver/README.md +++ b/receiver/couchdbreceiver/README.md @@ -4,7 +4,6 @@ This receiver fetches stats from a CouchDB server using the `/_node/{node-name}/_stats/couchdb` [endpoint](https://docs.couchdb.org/en/latest/api/server/common.html#node-node-name-stats). - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -51,4 +50,3 @@ The full list of settings exposed for this receiver are documented in [config.go ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) - diff --git a/receiver/datadogreceiver/README.md b/receiver/datadogreceiver/README.md index f4316fd293337..0cd89d3a89489 100644 --- a/receiver/datadogreceiver/README.md +++ b/receiver/datadogreceiver/README.md @@ -4,7 +4,6 @@ The Datadog receiver enables translation between Datadog and OpenTelemetry-compatible backends. It currently has support for Datadog's APM traces, metrics, and logs. - | Status | | | ------------- |-----------| | Stability | [alpha]: traces, metrics, logs | diff --git a/receiver/dockerstatsreceiver/README.md b/receiver/dockerstatsreceiver/README.md index 9c7bcf7cce84c..9d55543fec151 100644 --- a/receiver/dockerstatsreceiver/README.md +++ b/receiver/dockerstatsreceiver/README.md @@ -6,7 +6,6 @@ all desired running containers on a configured interval. These stats are for co resource usage of cpu, memory, network, and the [blkio controller](https://www.kernel.org/doc/Documentation/cgroup-v1/blkio-controller.txt). - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -124,7 +123,6 @@ For enhanced security, consider: For more information, see [issue #11791](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/11791). - ## Deprecations ### Transition to cpu utilization metric name aligned with OpenTelemetry specification diff --git a/receiver/elasticsearchreceiver/README.md b/receiver/elasticsearchreceiver/README.md index e2904546ef37b..3ed2a2bac9c48 100644 --- a/receiver/elasticsearchreceiver/README.md +++ b/receiver/elasticsearchreceiver/README.md @@ -6,7 +6,6 @@ This receiver queries the Elasticsearch [node stats](https://www.elastic.co/guid [index stats](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html) endpoints in order to scrape metrics from a running Elasticsearch cluster. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/envoyalsreceiver/README.md b/receiver/envoyalsreceiver/README.md index 6552dd1cebefe..7e228a5928c81 100644 --- a/receiver/envoyalsreceiver/README.md +++ b/receiver/envoyalsreceiver/README.md @@ -4,7 +4,6 @@ This is a receiver for the [Envoy gRPC ALS](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/access_loggers/grpc/v3/als.proto#envoy-v3-api-msg-extensions-access-loggers-grpc-v3-httpgrpcaccesslogconfig) sink. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | @@ -45,4 +44,3 @@ Other options can be configured to support more advanced use cases: - [gRPC settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configgrpc/README.md) including CORS - [TLS and mTLS settings](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md) - diff --git a/receiver/expvarreceiver/README.md b/receiver/expvarreceiver/README.md index 87140573548ce..4b4a15c7cd87d 100644 --- a/receiver/expvarreceiver/README.md +++ b/receiver/expvarreceiver/README.md @@ -1,12 +1,11 @@ # Expvar Receiver -An Expvar Receiver scrapes metrics from [expvar](https://pkg.go.dev/expvar), +An Expvar Receiver scrapes metrics from [expvar](https://pkg.go.dev/expvar), which exposes data in JSON format from an HTTP endpoint. The metrics are extracted from the `expvar` variable [memstats](https://pkg.go.dev/runtime#MemStats), which exposes various information about the Go runtime. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -19,11 +18,11 @@ which exposes various information about the Go runtime. [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -## Configuration +## Configuration ### Default -By default, without any configuration, a request will be sent to `http://localhost:8000/debug/vars` +By default, without any configuration, a request will be sent to `http://localhost:8000/debug/vars` every 60 seconds. The default configuration is achieved by the following: ```yaml @@ -37,8 +36,8 @@ The following can be configured: - Configure the HTTP client for scraping the expvar variables. The full set of configuration options for the client can be found in the core repo's [confighttp](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confighttp#client-configuration). - - defaults: - - `endpoint = http://localhost:8000/debug/vars` + - defaults: + - `endpoint = http://localhost:8000/debug/vars` - `timeout = 3s` - `collection_interval` - Configure how often the metrics are scraped. - default: 1m diff --git a/receiver/faroreceiver/README.md b/receiver/faroreceiver/README.md index b2583047b870e..a0bbe22fa8cc9 100644 --- a/receiver/faroreceiver/README.md +++ b/receiver/faroreceiver/README.md @@ -4,7 +4,6 @@ This receiver can receive telemetry data in [Faro format](https://github.com/grafana/faro) that adheres to the [Faro OpenAPI schema](https://github.com/grafana/faro/blob/main/spec/gen/faro.gen.yaml). - | Status | | | ------------- |-----------| | Stability | [alpha]: logs, traces | @@ -21,7 +20,6 @@ This receiver can receive telemetry data in [Faro format](https://github.com/gra Faro follows the [confighttp] configuration, some examples are shown below - ### Example Configuration ```yaml diff --git a/receiver/filelogreceiver/README.md b/receiver/filelogreceiver/README.md index 85382f4e46bb7..d633b85c289b1 100644 --- a/receiver/filelogreceiver/README.md +++ b/receiver/filelogreceiver/README.md @@ -3,7 +3,6 @@ This receiver tails and parses logs from files. - | Status | | | ------------- |-----------| | Stability | [beta]: logs | @@ -94,7 +93,7 @@ The `omit_pattern` setting can be used to omit the start/end pattern from each e ### Supported encodings -| Key | Description +| Key | Description | | --- | --- | | `nop` | No encoding validation. Treats the file as a stream of raw bytes | | `utf-8` | UTF-8 encoding | @@ -171,7 +170,7 @@ receivers: - Use `ignore` when you want to avoid duplicate logs and your log rotation strategy ensures that rotated files are properly renamed or moved. - Use `read_whole_file` when data completeness is critical and you can tolerate duplicate logs, or when you have deduplication logic downstream. -- Use `read_new` when files are expected to be deleted after rotation and you want to read only new data written after the truncation point. +- Use `read_new` when files are expected to be deleted after rotation and you want to read only new data written after the truncation point. ## Example - Tailing a simple json file diff --git a/receiver/flinkmetricsreceiver/README.md b/receiver/flinkmetricsreceiver/README.md index 555fdfc6926bd..7ba38330fc7b4 100644 --- a/receiver/flinkmetricsreceiver/README.md +++ b/receiver/flinkmetricsreceiver/README.md @@ -4,7 +4,6 @@ This receiver uses Flink's [REST API](https://nightlies.apache.org/flink/flink-docs-release-1.14/docs/ops/metrics/#rest-api-integration) to collect Jobmanager, Taskmanager, Job, Task and Operator metrics. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -48,4 +47,3 @@ The full list of settings exposed for this receiver are documented in [config.go ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) - diff --git a/receiver/fluentforwardreceiver/README.md b/receiver/fluentforwardreceiver/README.md index 84cfa3cb03d3f..7e940fba53194 100644 --- a/receiver/fluentforwardreceiver/README.md +++ b/receiver/fluentforwardreceiver/README.md @@ -4,7 +4,6 @@ This receiver runs a TCP server that accepts events via the [Fluent Forward protocol](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1). - | Status | | | ------------- |-----------| | Stability | [beta]: logs | diff --git a/receiver/githubreceiver/README.md b/receiver/githubreceiver/README.md index 77a94cb57a7e0..f0c6693e03ef0 100644 --- a/receiver/githubreceiver/README.md +++ b/receiver/githubreceiver/README.md @@ -115,7 +115,6 @@ see the [Scraping README][ghsread]. [ghsread]: internal/scraper/githubscraper/README.md#github-limitations - ### GitHub Personal Access Token (PAT) Setup To create a GitHub Personal Access Token (PAT), please refer to the official [documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens). @@ -123,7 +122,7 @@ To create a GitHub Personal Access Token (PAT), please refer to the official [do **Organization or Personal Access:** When generating the PAT, select the appropriate `Resource owner` — either your personal account or the organization and choose the correct `Repository access` type. For fine-grained tokens, explicitly configure the necessary `Repository permissions` or `Organization permissions`. -**Note**: +**Note**: The PAT must have read access to the target repositories. If the PAT doesn't have permission to access repositories in the target organization, only the repository count metric will be available. Detailed repository metrics cannot be fetched. ## Traces - Getting Started diff --git a/receiver/githubreceiver/internal/scraper/githubscraper/README.md b/receiver/githubreceiver/internal/scraper/githubscraper/README.md index 3de7d5ae6d4a7..bb5bc3f727c20 100644 --- a/receiver/githubreceiver/internal/scraper/githubscraper/README.md +++ b/receiver/githubreceiver/internal/scraper/githubscraper/README.md @@ -126,7 +126,6 @@ scrapers: ## Branch Data Limitations - Due to the limitations of the GitHub GraphQL and REST APIs, some data retrieved may not be as expected. Notably there are spots in the code which link to this section that make decisions based on these limitations. @@ -137,7 +136,7 @@ place of `AheadBy` and vice versa. This is a byproduct of the `getBranchData` query which returns all the refs (branches) from a given repository and the comparison to the default branch (trunk). Comparing it here reduces the need to make a query that gets all the names of the refs (branches), and then queries -against each branch. +against each branch. Another such byproduct of this method is the skipping of metric creation if the branch is the default branch (trunk) or if no changes have been made to the diff --git a/receiver/gitlabreceiver/README.md b/receiver/gitlabreceiver/README.md index 6b1dcbda9887d..a7269969e87fc 100644 --- a/receiver/gitlabreceiver/README.md +++ b/receiver/gitlabreceiver/README.md @@ -19,7 +19,7 @@ This is accomplished through the processing of GitLab webhook events for pipelines. The [`pipeline`](https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#pipeline-events) event payloads are then constructed into `trace` telemetry. -Each GitLab pipeline, along with its jobs, is converted +Each GitLab pipeline, along with its jobs, is converted into trace spans, allowing the observation of workflow execution times, success, and failure rates. diff --git a/receiver/googlecloudmonitoringreceiver/README.md b/receiver/googlecloudmonitoringreceiver/README.md index eb24794cb03ea..5e843792a7659 100644 --- a/receiver/googlecloudmonitoringreceiver/README.md +++ b/receiver/googlecloudmonitoringreceiver/README.md @@ -4,7 +4,6 @@ The primary objective of the Google Cloud Monitoring Receiver is to gather time series metrics data from all Google services and convert this data into a pipeline format that facilitates further use. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/receiver/googlecloudpubsubreceiver/README.md b/receiver/googlecloudpubsubreceiver/README.md index a2bd527cefab2..f5e05bca227d4 100644 --- a/receiver/googlecloudpubsubreceiver/README.md +++ b/receiver/googlecloudpubsubreceiver/README.md @@ -3,7 +3,6 @@ This receiver gets OTLP messages from a Google Cloud [Pubsub](https://cloud.google.com/pubsub) subscription. - | Status | | | ------------- |-----------| | Stability | [beta]: traces, logs, metrics | @@ -17,16 +16,16 @@ This receiver gets OTLP messages from a Google Cloud [Pubsub](https://cloud.goog > ⚠️ This is a community-provided module. It has been developed and extensively tested at Collibra, but it is not officially supported by GCP. - + The following configuration options are supported: * `project` (Optional): The Google Cloud Project of the client connects to. * `subscription` (Required): The subscription name to receive OTLP data from. The subscription name should be a fully qualified resource name (eg: `projects/otel-project/subscriptions/otlp`). * `encoding` (Optional): The encoding that will be used to received data from the subscription. This can either be - `otlp_proto_trace`, `otlp_proto_metric`, `otlp_proto_log` or and encoding extension (see `encoding`). This will only + `otlp_proto_trace`, `otlp_proto_metric`, `otlp_proto_log` or and encoding extension (see `encoding`). This will only be used as a fallback, when no `content-type` attribute is present. -* `compression` (Optional): The compression that will be used on received data from the subscription. When set it can +* `compression` (Optional): The compression that will be used on received data from the subscription. When set it can only be `gzip`. This will only be used as a fallback, when no `content-encoding` attribute is present. * `endpoint` (Optional): Override the default Pubsub Endpoint, useful when connecting to the PubSub emulator instance or switching between [global and regional service endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints). @@ -50,7 +49,7 @@ receivers: The `encoding` options allows you to specify Encoding Extensions for decoding messages on the subscription. An extension need to be configured in the `extensions` section, and added to pipeline in the collectors configuration file. -The following example shows how to use the text encoding extension for ingesting arbitrary text message on a +The following example shows how to use the text encoding extension for ingesting arbitrary text message on a subscription, wrapping them in OTLP Log messages. Note that not all extensions support all signals. ```yaml @@ -68,7 +67,7 @@ service: exporters: [debug] ``` -The receiver also supports build in encodings for the native OTLP Protobuf encodings, without the need to specify an +The receiver also supports build in encodings for the native OTLP Protobuf encodings, without the need to specify an Encoding Extensions. | encoding | description | @@ -95,8 +94,8 @@ When no encoding is specified, the receiver will try to discover the type of the ## Pubsub subscription -The Google Cloud [Pubsub](https://cloud.google.com/pubsub) receiver doesn't automatically create subscriptions, -it expects the subscription to be created upfront. Security wise it's best to give the collector its own +The Google Cloud [Pubsub](https://cloud.google.com/pubsub) receiver doesn't automatically create subscriptions, +it expects the subscription to be created upfront. Security wise it's best to give the collector its own service account and give the subscription `Pub/Sub Subscriber` permission. The subscription should also be of delivery type `Pull`. @@ -110,7 +109,7 @@ The flow control allows to fine tune the flow control settings for the subscript * `stream_ack_deadline` (Optional): The ack deadline to use for the stream. The minimum deadline you can specify is 10 seconds. The maximum deadline you can specify is 600 seconds (10 minutes). * `max_outstanding_messages` (Optional): Flow control settings for the maximum number of outstanding messages. When - there are max_outstanding_messages currently sent to the streaming pull client that have not yet been acked or + there are max_outstanding_messages currently sent to the streaming pull client that have not yet been acked or nacked, the server stops sending more messages. The sending of messages resumes once the number of outstanding messages is less than this value. If the value is <= 0, there is no limit to the number of outstanding messages. * `max_outstanding_bytes` (Optional): Flow control settings for the maximum number of outstanding bytes. When there @@ -132,13 +131,12 @@ More details can be found at [Pub/Sub gRPC StreamingPullRequest](https://docs.cl ### Filtering When the messages on the subscription are accompanied by the correct attributes and you only need a specific -type in your pipeline, the messages can be [filtered](https://cloud.google.com/pubsub/docs/filtering) on the +type in your pipeline, the messages can be [filtered](https://cloud.google.com/pubsub/docs/filtering) on the subscription saving on egress fees. -An example of filtering on trace message only: +An example of filtering on trace message only: ``` attributes.ce-type = "org.opentelemetry.otlp.traces.v1" AND attributes.content-type = "application/protobuf" ``` - diff --git a/receiver/googlecloudpubsubreceiver/documentation.md b/receiver/googlecloudpubsubreceiver/documentation.md index 94a6426fcd83c..91bd463944840 100644 --- a/receiver/googlecloudpubsubreceiver/documentation.md +++ b/receiver/googlecloudpubsubreceiver/documentation.md @@ -13,7 +13,6 @@ Number of times a message couldn't be decoded by the configured encoder The receiver reads messages from Google Cloud Pub/Sub and tries to decode the message using the configured encoder. Each time a message fails to decoded the counter is increased. - | Unit | Metric Type | Value Type | Monotonic | Stability | | ---- | ----------- | ---------- | --------- | --------- | | 1 | Sum | Int | true | Development | @@ -26,7 +25,6 @@ The receiver uses the Google Cloud Pub/Sub StreamingPull API and keeps a open co recurrently close the connection after a time period to avoid a long-running sticky connection. This metric counts the number of the resets that occurred during the lifetime of the container. - | Unit | Metric Type | Value Type | Monotonic | Stability | | ---- | ----------- | ---------- | --------- | --------- | | 1 | Sum | Int | true | Development | diff --git a/receiver/googlecloudspannerreceiver/README.md b/receiver/googlecloudspannerreceiver/README.md index f3a00531e9112..b111028a27203 100644 --- a/receiver/googlecloudspannerreceiver/README.md +++ b/receiver/googlecloudspannerreceiver/README.md @@ -85,4 +85,3 @@ Brief description of configuration properties: - **instances** - list of Google Cloud Spanner instance for connection - **instance_id** - identifier of Google Cloud Spanner instance - **databases** - list of databases used from this instance - diff --git a/receiver/googlecloudspannerreceiver/cardinality.md b/receiver/googlecloudspannerreceiver/cardinality.md index 4df7193e9fff9..b9ba5ae4c9eb5 100644 --- a/receiver/googlecloudspannerreceiver/cardinality.md +++ b/receiver/googlecloudspannerreceiver/cardinality.md @@ -29,7 +29,6 @@ We have one time series per minute, meaning that we can have up to 1,440 (24 hou Taking into account 1440 time series we'll obtain **7691 / 1440 = 5** new time series per minute. - ## Labels For each metric in the Total N table, the labels that create a unique time series are **Project ID + Instance ID + Database ID**. diff --git a/receiver/haproxyreceiver/README.md b/receiver/haproxyreceiver/README.md index 7f356e5f33942..130c963e58ad3 100644 --- a/receiver/haproxyreceiver/README.md +++ b/receiver/haproxyreceiver/README.md @@ -3,7 +3,6 @@ The HAProxy receiver generates metrics by polling periodically the HAProxy process through a dedicated socket or HTTP URL. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -31,7 +30,7 @@ Default: 1 minute ### Initial delay settings (optional) defines how long this receiver waits before starting. -Default: `1s` +Default: `1s` ### Example configuration diff --git a/receiver/hostmetricsreceiver/README.md b/receiver/hostmetricsreceiver/README.md index 12e93d1c0bf1b..8337e19fc83b6 100644 --- a/receiver/hostmetricsreceiver/README.md +++ b/receiver/hostmetricsreceiver/README.md @@ -5,7 +5,6 @@ The Host Metrics receiver generates metrics about the host system scraped from various sources and host entity event as log. This is intended to be used when the collector is deployed as an agent. - | Status | | | ------------- |-----------| | Stability | [development]: logs | diff --git a/receiver/httpcheckreceiver/README.md b/receiver/httpcheckreceiver/README.md index 78c6ad74a4999..32cb2350ddca7 100644 --- a/receiver/httpcheckreceiver/README.md +++ b/receiver/httpcheckreceiver/README.md @@ -3,7 +3,6 @@ The HTTP Check Receiver can be used for synthetic checks against HTTP endpoints. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -182,7 +181,6 @@ receivers: - `max_size` / `min_size`: Response body size limits - `regex`: Regular expression matching - ### Example Configuration ```yaml diff --git a/receiver/huaweicloudcesreceiver/README.md b/receiver/huaweicloudcesreceiver/README.md index a6a77df27863a..d636f384efa5c 100644 --- a/receiver/huaweicloudcesreceiver/README.md +++ b/receiver/huaweicloudcesreceiver/README.md @@ -6,7 +6,6 @@ This receiver contains the implementation of the Huawei Cloud Collector. The receiver collects metrics from Huawei Cloud's CES service and sends them to the OpenTelemetry Collector for processing and exporting. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -28,12 +27,12 @@ The following settings are required: - `project_id`: The ID of the project in Huawei Cloud. This is used to identify which project's metrics are to be collected. See [Obtaining a Project ID](https://support.huaweicloud.com/intl/en-us/devg-apisign/api-sign-provide-proid.html). - `period`: The aggregation granularity of metrics retrieved from CES in seconds. For details about the aggregation, see [What Is Rollup?](https://support.huaweicloud.com/intl/en-us/ces_faq/ces_faq_0009.html). Possible values are 1, 300, 1200, 3600, 14400, and 86400. - - 1: Cloud Eye performs no aggregation and displays raw data. - - 300: Cloud Eye aggregates data every 5 minutes. - - 1200: Cloud Eye aggregates data every 20 minutes. - - 3600: Cloud Eye aggregates data every hour. - - 14400: Cloud Eye aggregates data every 4 hours. - - 86400: Cloud Eye aggregates data every 24 hours. + - 1: Cloud Eye performs no aggregation and displays raw data. + - 300: Cloud Eye aggregates data every 5 minutes. + - 1200: Cloud Eye aggregates data every 20 minutes. + - 3600: Cloud Eye aggregates data every hour. + - 14400: Cloud Eye aggregates data every 4 hours. + - 86400: Cloud Eye aggregates data every 24 hours. - `filter`: The filter field determines the aggregation method used for the metrics. This aggregation is applied to the data points within the specified period. Valid values for filter include: - `average`: Calculates the average value over the period. @@ -82,7 +81,6 @@ The full list of settings exposed for this receiver are documented in [config.go ### Huawei Cloud SDK Authentication Setup - To ensure secure authentication, the Access Key (AK) and Secret Key (SK) used by the Huawei Cloud SDK must be stored in environment variables. See [Obtaining an AK/SK](https://support.huaweicloud.com/intl/en-us/devg-apisign/api-sign-provide-aksk.html). Before running the application, you need to set the environment variables `HUAWEICLOUD_SDK_AK` and `HUAWEICLOUD_SDK_SK` in your local environment. Here’s how you can do it: @@ -103,15 +101,13 @@ Before running the application, you need to set the environment variables `HUAWE ``` ## Error handling -If you encounter any errors, please refer to: +If you encounter any errors, please refer to: - [Huawei Cloud Error Codes](https://support.huaweicloud.com/intl/en-us/devg-apisign/api-sign-errorcode.html) - [CES Error Codes](https://support.huaweicloud.com/intl/en-us/api-ces/ErrorCode.html) - [Quota management](https://support.huaweicloud.com/intl/en-us/usermanual-ces/en-us_topic_0154940152.html) - ## Converting CES metric representation to OpenTelemetry metric representation - | Source Field | Target Field | Description | |--------------------------|----------------------------------------------|-------------------------------------------------------------------------------------------------------| | **metric_name** | `scoped_metric.metric.name` | The name of the metric. | @@ -126,7 +122,6 @@ If you encounter any errors, please refer to: | *N/A* | `resource.attributes["cloud.provider"]` | Set to `"huawei_cloud"` as the cloud provider. | | *N/A* | `scoped_metric.scope.name` | Set to `"huawei_cloud_ces"` as the scope name. | | *N/A* | `scoped_metric.scope.version` | Set to `"v1"` as the scope version. | - | ### Notes diff --git a/receiver/icmpcheckreceiver/README.md b/receiver/icmpcheckreceiver/README.md index a02c31d4e3fcc..3e175fa373d88 100644 --- a/receiver/icmpcheckreceiver/README.md +++ b/receiver/icmpcheckreceiver/README.md @@ -5,7 +5,6 @@ The ICMP Check Receiver can be used for synthetic checks against network endpoin Requests (pings) to the specified `targets`. It provides device availability monitoring and interface statistics with native OpenTelemetry OTLP metrics generation. - | Status | | | ------------- |-----------| | Stability | [development]: metrics | @@ -31,7 +30,7 @@ Each target has the following properties: - `host` (required): A hostname or IP address to be pinged. - `ping_count` (optional, default = `3`): The number of packets after which the pinger is stopped. -- `ping_interval` (optional, default = `1s`): The wait time between each packet send. +- `ping_interval` (optional, default = `1s`): The wait time between each packet send. - `ping_timeout` (optional, default = `5s`): Specifies a timeout before ping exits, regardless of how many packets have been received. ### Optional Metrics Configuration diff --git a/receiver/iisreceiver/README.md b/receiver/iisreceiver/README.md index c2f771e4fe522..19e2e245c4d0a 100644 --- a/receiver/iisreceiver/README.md +++ b/receiver/iisreceiver/README.md @@ -4,7 +4,6 @@ The Microsoft IIS Receiver grabs metrics about an IIS instance using the Windows Performance Counters. Because of this, it is a Windows only receiver. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/influxdbreceiver/README.md b/receiver/influxdbreceiver/README.md index deccb67c0adcb..b4944ecb26919 100644 --- a/receiver/influxdbreceiver/README.md +++ b/receiver/influxdbreceiver/README.md @@ -3,7 +3,6 @@ This receiver accepts metrics data as [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/). - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -81,4 +80,3 @@ prometheus,quantile=0.9 rpc_duration_seconds=9001 prometheus,quantile=0.99 rpc_duration_seconds=76656 prometheus rpc_duration_seconds_count=1.7560473e+07,rpc_duration_seconds_sum=2693 ``` - diff --git a/receiver/jaegerreceiver/README.md b/receiver/jaegerreceiver/README.md index a0de2f99c077b..44b5d5255d42c 100644 --- a/receiver/jaegerreceiver/README.md +++ b/receiver/jaegerreceiver/README.md @@ -3,7 +3,6 @@ The Jaeger Receiver receives trace data in the [Jaeger](https://www.jaegertracing.io/) format. - | Status | | | ------------- |-----------| | Stability | [beta]: traces | @@ -75,4 +74,3 @@ Several helper files are leveraged to provide additional capabilities automatica ## Remote Sampling Since version [v0.61.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.61.0), remote sampling is no longer supported by the jaeger receiver. Since version [v0.59.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.59.0), the [jaegerremotesapmpling](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.61.0/extension/jaegerremotesampling/README.md) extension is available that can be used instead. - diff --git a/receiver/jmxreceiver/README.md b/receiver/jmxreceiver/README.md index 851683405df5f..285b5ef69951b 100644 --- a/receiver/jmxreceiver/README.md +++ b/receiver/jmxreceiver/README.md @@ -268,4 +268,3 @@ Corresponds to the `otel.resource.attributes` property. SLF4J log level for the JMX metrics gatherer. Must be one of: `"trace"`, `"debug"`, `"info"`, `"warn"`, `"error"`, `"off"`. If not provided, will attempt to match to the current log level of the collector. Corresponds to the `org.slf4j.simpleLogger.defaultLogLevel` property. - diff --git a/receiver/k8sclusterreceiver/README.md b/receiver/k8sclusterreceiver/README.md index 8758d03e886b4..52269e95681da 100644 --- a/receiver/k8sclusterreceiver/README.md +++ b/receiver/k8sclusterreceiver/README.md @@ -5,7 +5,6 @@ The Kubernetes Cluster receiver collects cluster-level metrics and entity events API server. It uses the K8s API to listen for updates. A single instance of this receiver should be used to monitor a cluster. - | Status | | | ------------- |-----------| | Stability | [development]: logs | @@ -128,7 +127,6 @@ k8s_cluster: ... ``` - ### metadata_exporters A list of metadata exporters to which metadata being collected by this receiver @@ -159,7 +157,7 @@ See [experimentalmetricmetadata/metadata.go](https://github.com/open-telemetry/o The same metadata will be also emitted as entity events in the form of log records if this receiver is connected to a logs pipeline. See [opentelemetry-collector-contrib#23565](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23565) -for the format of emitted log records. +for the format of emitted log records. ## Compatibility @@ -222,7 +220,7 @@ EOF ### RBAC -Use the below commands to create a `ClusterRole` with required permissions and a +Use the below commands to create a `ClusterRole` with required permissions and a `ClusterRoleBinding` to grant the role to the service account created above. ```bash @@ -323,7 +321,7 @@ EOF ``` As an alternative to setting up a `ClusterRole`/`ClusterRoleBinding`, it is also possible to limit the observed resources to a list of -particular namespaces by setting the `namespaces` option of the receiver. This allows the collector to only rely on `Roles`/`RoleBindings`, +particular namespaces by setting the `namespaces` option of the receiver. This allows the collector to only rely on `Roles`/`RoleBindings`, instead of granting the collector cluster-wide read access to resources. Note however, that in this case the following cluster-scoped resources will not be observed by the `k8sclusterreceiver`: @@ -488,4 +486,3 @@ Add the following rules to your ClusterRole: - list - watch ``` - diff --git a/receiver/k8seventsreceiver/README.md b/receiver/k8seventsreceiver/README.md index 517efef8b99ef..9e86fa683924f 100644 --- a/receiver/k8seventsreceiver/README.md +++ b/receiver/k8seventsreceiver/README.md @@ -4,7 +4,6 @@ The Kubernetes Events Receiver collects events from the Kubernetes API server. It collects all the new or updated events that come in. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | diff --git a/receiver/k8sobjectsreceiver/README.md b/receiver/k8sobjectsreceiver/README.md index d58b79bd59c01..200a301cdf975 100644 --- a/receiver/k8sobjectsreceiver/README.md +++ b/receiver/k8sobjectsreceiver/README.md @@ -76,7 +76,6 @@ this case, it will select `v1` by default. > - **Network-attached volumes** (`ReadWriteMany`): the volume is accessible from any node, so the collector pod can be freely rescheduled or fail over to a different node while still resuming from the correct resourceVersion. This is the recommended approach, especially when used with `k8s_leader_elector`. > - **Block volumes** (`ReadWriteOnce`): supported for single-replica deployments where restarts are graceful. Not recommended with leader election across multiple nodes, as Kubernetes may take 30–90 seconds to detach and reattach the volume after a node failure. - The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). diff --git a/receiver/kafkametricsreceiver/README.md b/receiver/kafkametricsreceiver/README.md index 16cf1ae7bcc01..14553c68cc8ca 100644 --- a/receiver/kafkametricsreceiver/README.md +++ b/receiver/kafkametricsreceiver/README.md @@ -4,7 +4,6 @@ The Kafka Metrics Receiver collects kafka metrics (brokers, topics, partitions, consumer groups) from a kafka server, and converts them into otlp. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/kafkareceiver/README.md b/receiver/kafkareceiver/README.md index 771cf857fea8a..84c8678e36bb1 100644 --- a/receiver/kafkareceiver/README.md +++ b/receiver/kafkareceiver/README.md @@ -5,7 +5,6 @@ The Kafka Receiver receives telemetry data from Kafka, with configurable topics with the `kafkaexporter` configured with `include_metadata_keys`, The Kafka Receiver will also propagate the Kafka headers to the downstream pipeline, giving access to the rest of the pipeline to arbitrary metadata keys and values. - | Status | | | ------------- |-----------| | Stability | [development]: profiles | diff --git a/receiver/kafkareceiver/documentation.md b/receiver/kafkareceiver/documentation.md index 4e3a44b122db6..6e925828543c1 100644 --- a/receiver/kafkareceiver/documentation.md +++ b/receiver/kafkareceiver/documentation.md @@ -236,7 +236,6 @@ Note that this metric may slow down high-volume consuming. Only produced when franz-go is enabled. This metric is reported with an assumption that the exporter and the receiver clocks are synchronized. - | Unit | Metric Type | Value Type | Stability | | ---- | ----------- | ---------- | --------- | | s | Histogram | Double | Development | diff --git a/receiver/kubeletstatsreceiver/README.md b/receiver/kubeletstatsreceiver/README.md index 69116d8fbe791..2f1a9c96ef85b 100644 --- a/receiver/kubeletstatsreceiver/README.md +++ b/receiver/kubeletstatsreceiver/README.md @@ -4,7 +4,6 @@ The Kubelet Stats Receiver pulls node, pod, container, and volume metrics from the API server on a kubelet and sends it down the metric pipeline for further processing. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/libhoneyreceiver/README.md b/receiver/libhoneyreceiver/README.md index e1ce2124cf30f..06f22d3430a70 100644 --- a/receiver/libhoneyreceiver/README.md +++ b/receiver/libhoneyreceiver/README.md @@ -4,7 +4,6 @@ The Libhoney Receiver will accept Trace and Logs signals that are emitted from applications that are instrumented using [Libhoney](https://docs.honeycomb.io/send-data/logs/structured/libhoney/) libraries. - | Status | | | ------------- |-----------| | Stability | [alpha]: traces, logs | @@ -95,7 +94,7 @@ receivers: ### Telemetry data types supported It will subscribe to the Traces and Logs signals but accept traffic destined for either pipeline using one http receiver -component. Libhoney does not differentiate between the two so the receiver will identify which pipeline to deliver the +component. Libhoney does not differentiate between the two so the receiver will identify which pipeline to deliver the spans or log records to. No support for metrics since they'd look just like logs. diff --git a/receiver/lokireceiver/README.md b/receiver/lokireceiver/README.md index 41c558dd1cbe6..11f110a4c2d23 100644 --- a/receiver/lokireceiver/README.md +++ b/receiver/lokireceiver/README.md @@ -7,7 +7,6 @@ It allows Promtail instances to specify the open telemetry collector as their lo This receiver runs HTTP and GRPC servers to ingest log entries in Loki format. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | diff --git a/receiver/macosunifiedloggingreceiver/README.md b/receiver/macosunifiedloggingreceiver/README.md index 5fcee2c1440ca..6869e903a37bb 100644 --- a/receiver/macosunifiedloggingreceiver/README.md +++ b/receiver/macosunifiedloggingreceiver/README.md @@ -4,7 +4,6 @@ The macOS Unified Logging Receiver collects logs from macOS systems using the native `log` command. This receiver supports both live system logs and archived log files (`.logarchive`). - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | @@ -77,7 +76,6 @@ receivers: format: "ndjson" ``` - ### With Filtering ```yaml @@ -97,7 +95,6 @@ receivers: max_log_age: 24h ``` - ## Predicate Examples Filter by subsystem: @@ -179,4 +176,3 @@ service: receivers: [macosunifiedlogging] exporters: [file] ``` - diff --git a/receiver/memcachedreceiver/README.md b/receiver/memcachedreceiver/README.md index 7cb79954df924..4309e863406c1 100644 --- a/receiver/memcachedreceiver/README.md +++ b/receiver/memcachedreceiver/README.md @@ -6,7 +6,6 @@ command](https://github.com/memcached/memcached/wiki/Commands#statistics). A detailed description of all the stats available is at https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L1159. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -55,4 +54,3 @@ with detailed sample configurations in [testdata/config.yaml](./testdata/config. ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) with further documentation in [documentation.md](./documentation.md) - diff --git a/receiver/mongodbatlasreceiver/README.md b/receiver/mongodbatlasreceiver/README.md index e16cf2a4f1d56..76a05eec5992f 100644 --- a/receiver/mongodbatlasreceiver/README.md +++ b/receiver/mongodbatlasreceiver/README.md @@ -1,12 +1,11 @@ # MongoDB Atlas Receiver -The MongoDB Atlas Receiver receives metrics from [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) +The MongoDB Atlas Receiver receives metrics from [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) via their [monitoring APIs](https://docs.atlas.mongodb.com/reference/api/monitoring-and-logs/), as well as alerts via a configured [webhook](https://www.mongodb.com/docs/atlas/tutorial/third-party-service-integrations/) and events from [events APIs](https://www.mongodb.com/docs/atlas/reference/api/events/). - | Status | | | ------------- |-----------| | Stability | [beta]: metrics, logs | @@ -21,7 +20,7 @@ and events from [events APIs](https://www.mongodb.com/docs/atlas/reference/api/e ## Getting Started -The MongoDB Atlas receiver takes the following parameters. `public_key` and +The MongoDB Atlas receiver takes the following parameters. `public_key` and `private_key` are the only two required values to receive metrics and logs and are obtained via the "API Keys" tab of the MongoDB Atlas Project Access Manager. In the example below both values are being pulled from the environment. diff --git a/receiver/mongodbreceiver/README.md b/receiver/mongodbreceiver/README.md index 428bd9c9cb87b..941b5d0ad9dd0 100644 --- a/receiver/mongodbreceiver/README.md +++ b/receiver/mongodbreceiver/README.md @@ -1,11 +1,10 @@ # MongoDB Receiver -This receiver fetches stats from a MongoDB instance using the +This receiver fetches stats from a MongoDB instance using the [golang mongo driver](https://github.com/mongodb/mongo-go-driver). Stats are collected via MongoDB's `dbStats` and `serverStatus` commands. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -230,5 +229,5 @@ See the [Collector feature gates](https://github.com/open-telemetry/opentelemetr **STABLE**: `receiver.mongodb.removeDatabaseAttr` -The feature gate `receiver.mongodb.removeDatabaseAttr` will remove the database name attribute from data points +The feature gate `receiver.mongodb.removeDatabaseAttr` will remove the database name attribute from data points because it is already found on the resource. This feature gate cannot be changed and will be removed soon. diff --git a/receiver/mysqlreceiver/README.md b/receiver/mysqlreceiver/README.md index b7afd70030837..ff7138208e715 100644 --- a/receiver/mysqlreceiver/README.md +++ b/receiver/mysqlreceiver/README.md @@ -18,7 +18,7 @@ This receiver queries MySQL's global status and InnoDB tables. Some metrics will not appear if their corresponding feature is inactive. -There are also optional metrics that you must specify in your configuration to collect, listed in [documentation.md](./documentation.md) +There are also optional metrics that you must specify in your configuration to collect, listed in [documentation.md](./documentation.md) ## Prerequisites @@ -48,7 +48,6 @@ GRANT SELECT ON performance_schema.* TO @'%'; ## Configuration - The following settings are optional: - `endpoint`: (default = `localhost:3306`) - `tls`: Defines the TLS configuration to use. If `tls` is not set, the default is to disable TLS connections. diff --git a/receiver/mysqlreceiver/documentation.md b/receiver/mysqlreceiver/documentation.md index 140135523f006..b8f72032f3a5c 100644 --- a/receiver/mysqlreceiver/documentation.md +++ b/receiver/mysqlreceiver/documentation.md @@ -684,7 +684,6 @@ events: Query sample collection enables monitoring of current running database statements. This provides real-time visibility into active queries, helping users monitor database activity and performance as part of their observability pipeline. - #### Attributes | Name | Description | Values | Semantic Convention | @@ -715,7 +714,6 @@ This provides real-time visibility into active queries, helping users monitor da Top query collection enables monitoring of the queries that consumed the most CPU in the database. This provides insights into query performance and resource usage, helping users identify and optimize high-impact queries as part of their observability pipeline. - #### Attributes | Name | Description | Values | Semantic Convention | diff --git a/receiver/namedpipereceiver/README.md b/receiver/namedpipereceiver/README.md index fdcaf5f9c3128..a5648362d1f99 100644 --- a/receiver/namedpipereceiver/README.md +++ b/receiver/namedpipereceiver/README.md @@ -3,7 +3,6 @@ This receiver supports opening a Unix Named Pipe (aka FIFO), and reading logs from it. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | diff --git a/receiver/netflowreceiver/README.md b/receiver/netflowreceiver/README.md index 2be91a5f9e792..07bcc54a8a84e 100644 --- a/receiver/netflowreceiver/README.md +++ b/receiver/netflowreceiver/README.md @@ -6,7 +6,6 @@ The Netflow Receiver can listen for [netflow](https://en.wikipedia.org/wiki/NetF data and convert it to OpenTelemetry logs. The receiver is based on the [goflow2](https://github.com/netsampler/goflow2) project. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | diff --git a/receiver/nginxreceiver/README.md b/receiver/nginxreceiver/README.md index d9d512bbb7011..449de31d103a8 100644 --- a/receiver/nginxreceiver/README.md +++ b/receiver/nginxreceiver/README.md @@ -4,7 +4,6 @@ This receiver can fetch stats from a NGINX instance using the `ngx_http_stub_status_module` module's `status` endpoint. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/nsxtreceiver/README.md b/receiver/nsxtreceiver/README.md index 27afe97ecab50..35b6e07700cec 100644 --- a/receiver/nsxtreceiver/README.md +++ b/receiver/nsxtreceiver/README.md @@ -4,7 +4,6 @@ This receiver fetches metrics important to run virtual networking using NSX-T. The receiver ingests metrics via the [NSX Rest API](https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/nsx_64_api.pdf). - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -79,4 +78,3 @@ The full list of settings exposed for this receiver are documented in [config.go ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) - diff --git a/receiver/ntpreceiver/README.md b/receiver/ntpreceiver/README.md index 17b65b6b4afc9..8f9a39adee1c0 100644 --- a/receiver/ntpreceiver/README.md +++ b/receiver/ntpreceiver/README.md @@ -3,7 +3,6 @@ This receiver periodically retrieves the clock offset from a NTP server. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -18,7 +17,7 @@ This receiver periodically retrieves the clock offset from a NTP server. ## Configuration -- `endpoint`: (default = `pool.ntp.org:123`) Endpoint of the NTP server. Must be formatted as `{host}:{port}`. +- `endpoint`: (default = `pool.ntp.org:123`) Endpoint of the NTP server. Must be formatted as `{host}:{port}`. - `collection_interval`: (default = `30m`): This receiver collects metrics on an interval. This value must be a string readable by Golang's [time.ParseDuration](https://pkg.go.dev/time#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. @@ -41,4 +40,3 @@ The full list of settings exposed for this receiver are documented in [config.go ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) - diff --git a/receiver/oracledbreceiver/README.md b/receiver/oracledbreceiver/README.md index e3e79bb60bab1..192856fbf02a0 100644 --- a/receiver/oracledbreceiver/README.md +++ b/receiver/oracledbreceiver/README.md @@ -3,7 +3,6 @@ This receiver periodically queries an Oracle Database host to collect metrics. - | Status | | | ------------- |-----------| | Stability | [development]: logs | @@ -95,7 +94,6 @@ receivers: ## Enabling events. - The following is a generic configuration that can be used for the default logs and metrics scraped by the Oracle DB receiver. diff --git a/receiver/osqueryreceiver/README.md b/receiver/osqueryreceiver/README.md index 6fc841866cae1..adf652e5c23ef 100644 --- a/receiver/osqueryreceiver/README.md +++ b/receiver/osqueryreceiver/README.md @@ -4,7 +4,6 @@ The osquery Receiver runs queries run on an [osquery](https://osquery.io/)'s daemon on a schedule and converts the output to logs. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | diff --git a/receiver/otelarrowreceiver/README.md b/receiver/otelarrowreceiver/README.md index f62ce5d306ed6..1433ce4fdcd20 100644 --- a/receiver/otelarrowreceiver/README.md +++ b/receiver/otelarrowreceiver/README.md @@ -7,7 +7,6 @@ Arrow](https://github.com/open-telemetry/otel-arrow) and standard https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md) protocol via gRPC. - | Status | | | ------------- |-----------| | Stability | [beta]: traces, metrics, logs | @@ -176,7 +175,7 @@ exporters: In addition to the standard [obsreport](https://pkg.go.dev/go.opentelemetry.io/collector/obsreport) -metrics, this component provides network-level measurement instruments +metrics, this component provides network-level measurement instruments measuring bytes transported. At the `normal` level of metrics detail: - `otelcol_receiver_recv`: uncompressed bytes received, prior to compression diff --git a/receiver/otelarrowreceiver/config.md b/receiver/otelarrowreceiver/config.md index 3f5ac4b9e1c8f..e8e57df0487a6 100644 --- a/receiver/otelarrowreceiver/config.md +++ b/receiver/otelarrowreceiver/config.md @@ -2,7 +2,6 @@ Config defines configuration for OTLP receiver. - ### Config | Name | Type | Default | Docs | @@ -95,5 +94,5 @@ Config defines configuration for OTLP receiver. | key_file | string | | Path to the TLS key to use for TLS required connections. (optional) | | client_ca_file | string | | Path to the TLS cert to use by the server to verify a client certificate. (optional) This sets the ClientCAs and ClientAuth to RequireAndVerifyClientCert in the TLSConfig. Please refer to https://godoc.org/crypto/tls#Config for more information. (optional) | -### time-Duration +### time-Duration An optionally signed sequence of decimal numbers, each with a unit suffix, such as `300ms`, `-1.5h`, or `2h45m`. Valid time units are `ns`, `us`, `ms`, `s`, `m`, `h`. diff --git a/receiver/otlpjsonfilereceiver/README.md b/receiver/otlpjsonfilereceiver/README.md index bee3ec4fe4c45..c0c83ee062e9d 100644 --- a/receiver/otlpjsonfilereceiver/README.md +++ b/receiver/otlpjsonfilereceiver/README.md @@ -7,7 +7,6 @@ This receiver will read pipeline data from JSON files. The data is written in using [OpenTelemetry protocol](https://github.com/open-telemetry/opentelemetry-proto). - | Status | | | ------------- |-----------| | Stability | [development]: profiles | diff --git a/receiver/podmanreceiver/README.md b/receiver/podmanreceiver/README.md index ac4f11151bd77..103f8cbebdc71 100644 --- a/receiver/podmanreceiver/README.md +++ b/receiver/podmanreceiver/README.md @@ -5,7 +5,6 @@ The Podman Stats Receiver queries the Podman service API to fetch stats for all on a configured interval. These stats are for container resource usage of cpu, memory, network, and the [blkio controller](https://www.kernel.org/doc/Documentation/cgroup-v1/blkio-controller.txt). - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -21,7 +20,6 @@ the [blkio controller](https://www.kernel.org/doc/Documentation/cgroup-v1/blkio- > :information_source: Requires Podman API version 3.3.1+ and Windows is not supported. - ## Configuration The following settings are required: @@ -77,17 +75,17 @@ receivers: The receiver emits the following metrics: - container.memory.usage.limit - container.memory.usage.total - container.memory.percent - container.network.io.usage.tx_bytes - container.network.io.usage.rx_bytes - container.blockio.io_service_bytes_recursive.write - container.blockio.io_service_bytes_recursive.read - container.cpu.usage.system - container.cpu.usage.total - container.cpu.percent - container.cpu.usage.percpu + container.memory.usage.limit + container.memory.usage.total + container.memory.percent + container.network.io.usage.tx_bytes + container.network.io.usage.rx_bytes + container.blockio.io_service_bytes_recursive.write + container.blockio.io_service_bytes_recursive.read + container.cpu.usage.system + container.cpu.usage.total + container.cpu.percent + container.cpu.usage.percpu See [./documentation.md](./documentation.md) for full detail. @@ -104,4 +102,3 @@ Recommended build tags to use when including this receiver in your build: - `containers_image_openpgp` - `exclude_graphdriver_btrfs` - `exclude_graphdriver_devicemapper` - diff --git a/receiver/postgresqlreceiver/README.md b/receiver/postgresqlreceiver/README.md index 17a24dfeb4829..1adea78828e25 100644 --- a/receiver/postgresqlreceiver/README.md +++ b/receiver/postgresqlreceiver/README.md @@ -3,7 +3,6 @@ This receiver queries the PostgreSQL [statistics collector](https://www.postgresql.org/docs/13/monitoring-stats.html). - | Status | | | ------------- |-----------| | Stability | [development]: logs | @@ -79,7 +78,7 @@ The following settings are also optional and nested under `tls` to help configur - `initial_delay` (default = `1s`): defines how long this receiver waits before starting. ### Query Sample Collection -We provide functionality to collect the query sample from PostgreSQL. It will get historical query +We provide functionality to collect the query sample from PostgreSQL. It will get historical query from `pg_stat_activity`. To enable it, you will need the following configuration ``` ... @@ -88,7 +87,7 @@ from `pg_stat_activity`. To enable it, you will need the following configuration enabled: true ... ``` -By default, query sample collection is disabled, also note, to use it, you will need +By default, query sample collection is disabled, also note, to use it, you will need to grant the user you are using `pg_monitor`. Take the example from `testdata/integration/init.sql` ```sql @@ -96,7 +95,7 @@ GRANT pg_monitor TO otelu; ``` The following options are available: -- `max_rows_per_query`: (optional, default=1000) The max number of rows would return from the query +- `max_rows_per_query`: (optional, default=1000) The max number of rows would return from the query against `pg_stat_activity`. ### Top Query Collection @@ -109,9 +108,9 @@ We provide functionality to collect the most executed queries from PostgreSQL. I ... ``` -Along with those attributes, we will also report the query plan we gathered if it is possible. +Along with those attributes, we will also report the query plan we gathered if it is possible. -By default, top query collection is disabled, also note, to use it, you will need +By default, top query collection is disabled, also note, to use it, you will need to create the extension to every database. Take the example from `testdata/integration/02-create-extension.sh` ```sql @@ -119,15 +118,15 @@ CREATE EXTENSION IF NOT EXISTS pg_stat_statements; ``` The following options are available: -- `max_rows_per_query`: (optional, default=1000) The max number of rows would return from the query +- `max_rows_per_query`: (optional, default=1000) The max number of rows would return from the query against `pg_stat_statements`. - `top_n_query`: (optional, default=200) The maximum number of active queries to report (to the next consumer) in a single run. -- `max_explain_each_interval`: (optional, default=1000). The maximum number of explain query to be sent in each scrape interval. The top query -collection would not get the query plan directly. Instead, we need to mimic the query in the database and get the query plan from database +- `max_explain_each_interval`: (optional, default=1000). The maximum number of explain query to be sent in each scrape interval. The top query +collection would not get the query plan directly. Instead, we need to mimic the query in the database and get the query plan from database separately. This could lead some resources usage and limit this will reduce the impact on your database. - `query_plan_cache_size`: (optional, default=1000). The query plan cache size. Once we got explain for one query, we will store it in the cache. This defines the cache's size for query plan. -- `query_plan_cache_ttl`: (optional, default=1h). How long before the query plan cache got expired. Example values: `1m`, `1h`. +- `query_plan_cache_ttl`: (optional, default=1h). How long before the query plan cache got expired. Example values: `1m`, `1h`. - `collection_interval`: (optional, default=60s). This receiver can collect top_query metrics on an interval. If not provided then the global collection_interval takes effect. This value must be a string readable by Golang's [time.ParseDuration](https://pkg.go.dev/time#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. ### Example Configuration diff --git a/receiver/postgresqlreceiver/documentation.md b/receiver/postgresqlreceiver/documentation.md index 6c4c04cfcd3ee..e3488f47bb0b1 100644 --- a/receiver/postgresqlreceiver/documentation.md +++ b/receiver/postgresqlreceiver/documentation.md @@ -220,7 +220,6 @@ Age of the oldest WAL file. This metric requires WAL to be enabled with at least one replica. - | Unit | Metric Type | Value Type | Stability | | ---- | ----------- | ---------- | --------- | | s | Gauge | Int | Development | @@ -231,7 +230,6 @@ Time between flushing recent WAL locally and receiving notification that the sta This metric requires WAL to be enabled with at least one replica. - | Unit | Metric Type | Value Type | Stability | | ---- | ----------- | ---------- | --------- | | s | Gauge | Int | Development | @@ -377,7 +375,6 @@ Time between flushing recent WAL locally and receiving notification that the sta This metric requires WAL to be enabled with at least one replica. - | Unit | Metric Type | Value Type | Stability | | ---- | ----------- | ---------- | --------- | | s | Gauge | Double | Development | diff --git a/receiver/pprofreceiver/README.md b/receiver/pprofreceiver/README.md index b3e5d59bdb9a5..8746134dac397 100644 --- a/receiver/pprofreceiver/README.md +++ b/receiver/pprofreceiver/README.md @@ -3,7 +3,6 @@ The Pprof Receiver collects profiles from files specified with a glob pattern, from a remote endpoint or the running collector. - | Status | | | ------------- |-----------| | Stability | [alpha]: profiles | diff --git a/receiver/prometheusreceiver/DESIGN.md b/receiver/prometheusreceiver/DESIGN.md index 4c09c5c11ba63..6aadd27b5aa47 100644 --- a/receiver/prometheusreceiver/DESIGN.md +++ b/receiver/prometheusreceiver/DESIGN.md @@ -29,7 +29,6 @@ service. We shall be able to retain parity from the following two setups: 1. app -> prometheus -> metric-endpoint 2. app -> otelcol-with-prometheus-receiver -> otelcol-prometheus-exporter-metrics-endpoint - ## Prometheus Text Format Overview Prometheus text format is a line orient format. Each non-empty line, which @@ -126,7 +125,7 @@ shown in the flowchart below: It basically does the following things in turn: 1. make a http call to fetch data from the binding [target](#target)'s metrics endpoint with [scraper](#scraper) - 2. acquired a [storageAppender](#storageappender) instance with the [Appendable](#appendable) interface + 2. acquired a [storageAppender](#storageappender) instance with the [Appendable](#appendable) interface 3. feed the data to a textParser 4. parse and feed metric data points to storage appender 5. commit if success or rollback @@ -152,7 +151,7 @@ type Appender interface { } type ExemplarAppender interface { - AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) + AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) } ``` @@ -355,7 +354,6 @@ metrics := []*metricspb.Metric{ *Note: `startTimestamp` is the timestamp cached from the first scrape, `currentTimestamp` is the timestamp of the current scrape* - ### Gauge Gauge, as described in the [Prometheus Metric Types Document](https://prometheus.io/docs/concepts/metric_types/#gauge), > is a metric that represents a single numerical value that can arbitrarily go up and down @@ -608,8 +606,8 @@ metrics := []*metricspb.Metric{ Points: []*metricspb.Point{ {Timestamp: startTimestamp, Value: &metricspb.Point_SummaryValue{ SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 17.391350544}, - Count: &wrappers.Int64Value{Value: 52489}, + Sum: &wrappers.DoubleValue{Value: 17.391350544}, + Count: &wrappers.Int64Value{Value: 52489}, Snapshot: &metricspb.SummaryValue_Snapshot{ PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ {Percentile: 0.0, Value: 0.0001271}, @@ -655,8 +653,8 @@ metrics := []*metricspb.Metric{ Points: []*metricspb.Point{ {Timestamp: currentTimestamp, Value: &metricspb.Point_SummaryValue{ SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 17.491350544}, - Count: &wrappers.Int64Value{Value: 52490}, + Sum: &wrappers.DoubleValue{Value: 17.491350544}, + Count: &wrappers.Int64Value{Value: 52490}, Snapshot: &metricspb.SummaryValue_Snapshot{ PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ {Percentile: 0.0, Value: 0.0001271}, diff --git a/receiver/prometheusreceiver/README.md b/receiver/prometheusreceiver/README.md index ac4f147845d70..598dda2ccc259 100644 --- a/receiver/prometheusreceiver/README.md +++ b/receiver/prometheusreceiver/README.md @@ -3,7 +3,6 @@ The Prometheus Receiver receives metric data in [Prometheus](https://prometheus.io/) format. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -26,7 +25,7 @@ Note: This component is currently work in progress. It has several limitations and please don't use it if the following limitations are a concern: * Collector cannot auto-scale the scraping yet when multiple replicas of the - collector are run. + collector are run. * When running multiple replicas of the collector with the same config, it will scrape the targets multiple times. * Users need to configure each replica with different scraping configuration @@ -209,15 +208,14 @@ receivers: - targets: ['localhost:8080'] ``` - This feature applies to the most common integer counter histograms; gauge histograms are dropped. In case a metric has both the conventional (aka classic) buckets and also native histogram buckets, only the native histogram buckets will be taken into account to create the corresponding exponential histogram. To scrape the classic buckets instead use the [scrape option](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) `always_scrape_classic_histograms`. ## OpenTelemetry Operator -Additional to this static job definitions this receiver allows to query a list of jobs from the -OpenTelemetryOperators TargetAllocator or a compatible endpoint. +Additional to this static job definitions this receiver allows to query a list of jobs from the +OpenTelemetryOperators TargetAllocator or a compatible endpoint. ```yaml receivers: @@ -294,7 +292,6 @@ The API server hosts the same paths as the Prometheus agent-mode API. These incl More info about querying `/api/v1/` and the data format that is returned can be found in the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/querying/api/). - ## Feature gates See [documentation.md](./documentation.md) for the complete list of feature gates supported by this receiver. diff --git a/receiver/prometheusremotewritereceiver/README.md b/receiver/prometheusremotewritereceiver/README.md index 98a796e3c45f7..98443f6c7e936 100644 --- a/receiver/prometheusremotewritereceiver/README.md +++ b/receiver/prometheusremotewritereceiver/README.md @@ -4,7 +4,6 @@ This receiver implements the [Prometheus Remote Write 2.0 protocol](https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/). - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -96,7 +95,7 @@ In Prometheus Remote Write v2, this problem is solved since the time series are ### Summaries and Classic Histograms are unsupported -As mentioned in [Histogram Atomicity](#histogram-atomicity), Prometheus Classic Histograms are split into several separate time series and, for this reason, it is impossible to determine if the amount of buckets received are the complete set. +As mentioned in [Histogram Atomicity](#histogram-atomicity), Prometheus Classic Histograms are split into several separate time series and, for this reason, it is impossible to determine if the amount of buckets received are the complete set. Summaries suffer from the same problem, a working Summary is composed by several time series just like Classic Histograms. The only difference is that instead of bucket boundaries, these time series represent pre-calculated quantiles. Since the quantiles can be sent in separate Remote Write requests, it's impossible to determine if the amount of quantiles received are enough to generate a complete Summary. diff --git a/receiver/pulsarreceiver/README.md b/receiver/pulsarreceiver/README.md index b443fdcffeff6..48060b87622ba 100644 --- a/receiver/pulsarreceiver/README.md +++ b/receiver/pulsarreceiver/README.md @@ -3,7 +3,6 @@ The Pulsar Receiver receives logs, metrics, and traces from [Apache Pulsar](https://pulsar.apache.org/). - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics, traces, logs | @@ -40,7 +39,7 @@ The following settings can be optionally configured: - `oauth2` - `issuer_url`: - `client_id`: - - `audience`: + - `audience`: - `scope`: - `private_key`: Path to the private client credentials json file. Must contain `type`, `client_id`, `client_secret` and `issuer_url` fields. - `athenz` @@ -56,7 +55,6 @@ The following settings can be optionally configured: only be used if `insecure` is set to true. - `tls_allow_insecure_connection`: configure whether the Pulsar client accept untrusted TLS certificate from broker (default: false) - Example configuration: ```yaml receivers: @@ -73,4 +71,3 @@ receivers: tls_allow_insecure_connection: false tls_trust_certs_file_path: ca.pem ``` - diff --git a/receiver/purefareceiver/README.md b/receiver/purefareceiver/README.md index 027fb49da1f43..9a9a470e2f442 100644 --- a/receiver/purefareceiver/README.md +++ b/receiver/purefareceiver/README.md @@ -3,7 +3,6 @@ The Pure Storage FlashArray Receiver receives metrics from the Pure Storage FlashArray. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -22,7 +21,7 @@ The Pure Storage FlashArray Receiver receives metrics from the Pure Storage Flas The following settings are required: - `endpoint` (default: `http://127.0.0.0:9490/metrics/array`): The URL of the scraper selected endpoint. - `fa_array_name` (no default): The array's pretty name to be used as a metrics label. - - `namespace` (default:purefa): The selected Pure Storage OpenMetrics Namespace to query. + - `namespace` (default:purefa): The selected Pure Storage OpenMetrics Namespace to query. In the below examples array01 is using the [Pure Storage FlashArray OpenMetrics exporter](https://github.com/PureStorage-OpenConnect/pure-fa-openmetrics-exporter), while array02 is using the native on-box metrics provided in Purity//FA v6.6.11+. @@ -111,4 +110,3 @@ service: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/purefbreceiver/README.md b/receiver/purefbreceiver/README.md index 0ee7ba0533685..c18350ffef731 100644 --- a/receiver/purefbreceiver/README.md +++ b/receiver/purefbreceiver/README.md @@ -4,7 +4,6 @@ The Pure Storage FlashBlade receiver, receives metrics from Pure Storage FlashBlade via the [Pure Storage FlashBlade OpenMetrics Exporter](https://github.com/PureStorage-OpenConnect/pure-fb-openmetrics-exporter) - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -23,7 +22,7 @@ The Pure Storage FlashBlade receiver, receives metrics from Pure Storage FlashBl The following settings are required: - `endpoint` (default: `http://172.31.60.207:9491/metrics/array`): The URL of the scraper selected endpoint -### Important +### Important - Only endpoints explicitly added on will be scraped. e.g: `clients` @@ -56,4 +55,3 @@ receivers: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/rabbitmqreceiver/README.md b/receiver/rabbitmqreceiver/README.md index a5a6417d81599..004e107f955a4 100644 --- a/receiver/rabbitmqreceiver/README.md +++ b/receiver/rabbitmqreceiver/README.md @@ -3,7 +3,6 @@ This receiver fetches stats from a RabbitMQ node using the [RabbitMQ Management Plugin](https://www.rabbitmq.com/management.html). - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/receivercreator/README.md b/receiver/receivercreator/README.md index 0c2d88ceaa6f1..66d819e67d1b7 100644 --- a/receiver/receivercreator/README.md +++ b/receiver/receivercreator/README.md @@ -3,7 +3,6 @@ This receiver can instantiate other receivers at runtime based on whether observed endpoints match a configured rule. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs, traces, profiles | @@ -168,7 +167,6 @@ None See `redis/2` in [examples](#examples). - **receivers.<receiver_type/id>.resource_attributes** ```yaml @@ -478,7 +476,6 @@ service: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - ## Generate receiver configurations from provided Hints Note: When hints feature is enabled if hints are present for an endpoint no receiver templates will be evaluated. @@ -528,7 +525,6 @@ collecting metrics and logs signals from the target Pods/containers. `io.opentelemetry.discovery.metrics/scraper` (example: `"nginx"`) - #### Define configuration `io.opentelemetry.discovery.metrics/config` @@ -550,7 +546,6 @@ io.opentelemetry.discovery.metrics/config: | xyz: "abc" ``` - #### Support multiple target containers Users can target the annotation to a specific container by suffixing it with the name of the port that container exposes: @@ -567,7 +562,7 @@ the Pod level hints are used as a fallback (see detailed example below). The current implementation relies on the implementation of `k8sobserver` extension and specifically the [pod_endpoint](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.111.0/extension/observer/k8sobserver/pod_endpoint.go). -The hints are evaluated per container by extracting the annotations from each [`Port` endpoint](#Port) that is emitted. +The hints are evaluated per container by extracting the annotations from each [`Port` endpoint](#Port) that is emitted. ### Supported logs annotations @@ -635,7 +630,6 @@ The current implementation relies on the implementation of `k8sobserver` extensi the [pod_endpoint](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.111.0/extension/observer/k8sobserver/pod_endpoint.go). The hints are evaluated per container by extracting the annotations from each [`Pod Container` endpoint](#pod-container) that is emitted. - ### Examples #### Metrics and Logs example diff --git a/receiver/redfishreceiver/README.md b/receiver/redfishreceiver/README.md index d612fc24b6ae7..5095a03eb23a9 100644 --- a/receiver/redfishreceiver/README.md +++ b/receiver/redfishreceiver/README.md @@ -4,7 +4,6 @@ This receiver fetches metrics for a server running a [Redfish](https://en.wikipedia.org/wiki/Redfish_(specification)) API. - | Status | | | ------------- |-----------| | Stability | [development]: metrics | @@ -37,7 +36,7 @@ The following settings are optional: - `servers`: The list of redfish servers to be monitored. Each server has the following properties: -- `base_url` (required): base url to monitor in the form [https][://][host]:[port]. +- `base_url` (required): base url to monitor in the form `https://host:port`. - `username` (required): Redfish username. - `password` (required): Redfish password. - `insecure` (optional, default: `false`): Sets the protocol security. diff --git a/receiver/redisreceiver/README.md b/receiver/redisreceiver/README.md index 37479e07acb5b..fcbdb7b01aa3b 100644 --- a/receiver/redisreceiver/README.md +++ b/receiver/redisreceiver/README.md @@ -5,7 +5,6 @@ The Redis Receiver is designed to retrieve Redis INFO data from a single Redis instance, build metrics from that data, and send them to the next consumer at a configurable interval. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -34,13 +33,13 @@ The Redis receiver turns this data into a gauge... ```go func usedCPUSys() *redisMetric { - return &redisMetric{ - key: "used_cpu_sys", - name: "redis.cpu.time", - units: "s", - mdType: metricspb.MetricDescriptor_GAUGE_DOUBLE, - labels: map[string]string{"state": "sys"}, - } + return &redisMetric{ + key: "used_cpu_sys", + name: "redis.cpu.time", + units: "s", + mdType: metricspb.MetricDescriptor_GAUGE_DOUBLE, + labels: map[string]string{"state": "sys"}, + } } ``` @@ -97,4 +96,3 @@ receivers: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/redisreceiver/config.md b/receiver/redisreceiver/config.md index 48ca8477a84bb..c3ff66d664e42 100644 --- a/receiver/redisreceiver/config.md +++ b/receiver/redisreceiver/config.md @@ -4,7 +4,6 @@ ScraperControllerSettings defines common settings for a scraper controller configuration. Scraper controller receivers can embed this struct and extend it with more fields if needed. - ### redisreceiver-Config | Name | Field Info | Default | Docs | @@ -267,5 +266,5 @@ and extend it with more fields if needed. | ---- | --------- | ------- | ---- | | enabled |bool| true | | -### time-Duration +### time-Duration An optionally signed sequence of decimal numbers, each with a unit suffix, such as `300ms`, `-1.5h`, or `2h45m`. Valid time units are `ns`, `us`, `ms`, `s`, `m`, `h`. \ No newline at end of file diff --git a/receiver/riakreceiver/README.md b/receiver/riakreceiver/README.md index 3ccbb3e3a0a13..ea42bb45e5ee1 100644 --- a/receiver/riakreceiver/README.md +++ b/receiver/riakreceiver/README.md @@ -12,7 +12,6 @@ [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib - Riak metrics will be collected from the [/stats](https://docs.riak.com/riak/kv/2.2.3/developing/api/http/status) endpoint. @@ -46,5 +45,3 @@ receivers: ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) - - diff --git a/receiver/saphanareceiver/README.md b/receiver/saphanareceiver/README.md index 520278c3c6b2c..3c029a49c2bb0 100644 --- a/receiver/saphanareceiver/README.md +++ b/receiver/saphanareceiver/README.md @@ -4,7 +4,6 @@ This receiver can fetch stats from a SAP HANA instance. It leverages the [driver](https://github.com/SAP/go-hdb) written by SAP for connecting to SAP HANA with the golang sql module to execute several monitoring queries. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -102,4 +101,3 @@ with detailed sample configurations in [testdata/config.yaml](./testdata/config. Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml). Further details of the monitoring queries used to collect them may be found in [queries.go](./queries.go). > If all of the metrics collected by a given monitoring query are marked as `enabled: false` in the receiver configuration, the monitoring query will not be executed. - diff --git a/receiver/signalfxreceiver/README.md b/receiver/signalfxreceiver/README.md index 9a163bc147cf1..c03d35ff6adfc 100644 --- a/receiver/signalfxreceiver/README.md +++ b/receiver/signalfxreceiver/README.md @@ -75,4 +75,4 @@ service: ## Access token passthrough Access token passthrough is no longer supported, to achieve similar behavior configure your collector -to use the `headers_setter` extension to pass the access token. +to use the `headers_setter` extension to pass the access token. diff --git a/receiver/simpleprometheusreceiver/README.md b/receiver/simpleprometheusreceiver/README.md index 2b0fc39d5286b..0097bd7b6a1fd 100644 --- a/receiver/simpleprometheusreceiver/README.md +++ b/receiver/simpleprometheusreceiver/README.md @@ -4,7 +4,6 @@ This receiver provides a simple configuration interface to configure the prometheus receiver to scrape metrics from a single target. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | diff --git a/receiver/skywalkingreceiver/README.md b/receiver/skywalkingreceiver/README.md index ae730008b268b..63be9be1fa2b8 100644 --- a/receiver/skywalkingreceiver/README.md +++ b/receiver/skywalkingreceiver/README.md @@ -35,7 +35,6 @@ object configuration parameter. See our [security best practices doc](https://opentelemetry.io/docs/security/config-best-practices/#protect-against-denial-of-service-attacks) to understand how to set the endpoint in different environments. - Examples: ```yaml diff --git a/receiver/snmpreceiver/README.md b/receiver/snmpreceiver/README.md index 127862406e5d9..5a907c0d0e947 100644 --- a/receiver/snmpreceiver/README.md +++ b/receiver/snmpreceiver/README.md @@ -5,7 +5,6 @@ This receiver fetches stats from an SNMP enabled host using a [golang snmp client](https://github.com/gosnmp/gosnmp). Metrics are collected based upon different configurations in the config file. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -128,7 +127,7 @@ Attribute configurations are used to define what resource attributes will be use | Field Name | Description | Value | Default | | -- | -- | -- | -- | | `oid` | The SNMP scalar OID value to grab data from (must end in .0). | string | | -| `resource_attributes` | The names of the related resource attribute configurations, allowing scalar oid metrics to be added to resources that have one or more scalar oid resource attributes. Cannot have indexed resource attributes as values. | string[] | | +| `resource_attributes` | The names of the related resource attribute configurations, allowing scalar oid metrics to be added to resources that have one or more scalar oid resource attributes. Cannot have indexed resource attributes as values. | string[] | | | `attributes` | The names of the related attribute enum configurations as well as the values to attach to this returned SNMP scalar data. This can be used to have a metric config with multiple ScalarOIDs as different datapoints with different attribute values within the same metric | Attribute | | #### ColumnOID Configuration @@ -252,4 +251,3 @@ receivers: ``` The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/snowflakereceiver/README.md b/receiver/snowflakereceiver/README.md index ac29e65053577..1fda340662a35 100644 --- a/receiver/snowflakereceiver/README.md +++ b/receiver/snowflakereceiver/README.md @@ -3,7 +3,6 @@ This receiver collects metrics from a Snowflake account by connecting to and querying a Snowflake deployment. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/receiver/solacereceiver/README.md b/receiver/solacereceiver/README.md index 3578e0e48b287..b2f51cceb5309 100644 --- a/receiver/solacereceiver/README.md +++ b/receiver/solacereceiver/README.md @@ -3,7 +3,6 @@ The Solace Receiver receives trace data from a [Solace PubSub+ Event Broker](https://solace.com/products/event-broker/). - | Status | | | ------------- |-----------| | Stability | [beta]: traces | @@ -102,4 +101,3 @@ service: traces/solace: receivers: [solace/primary,solace/backup] ``` - diff --git a/receiver/splunkenterprisereceiver/README.md b/receiver/splunkenterprisereceiver/README.md index a270b201af2f8..d86db2eff7cfa 100644 --- a/receiver/splunkenterprisereceiver/README.md +++ b/receiver/splunkenterprisereceiver/README.md @@ -7,7 +7,6 @@ results from ad-hoc searches. Because of this, care must be taken by users when Cloud deployments. The primary purpose of this receiver is to empower those tasked with the maintenance and care of a Splunk Enterprise deployment to leverage opentelemetry and their observability toolset in their jobs. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -24,7 +23,7 @@ jobs. **By default the Splunk Enterprise receiver is not configured to gather any metrics other than `splunk.health`** -The following settings are required, omitting them will either cause your receiver to fail to compile or result in 4/5xx return codes during scraping. +The following settings are required, omitting them will either cause your receiver to fail to compile or result in 4/5xx return codes during scraping. **NOTE:** These must be set for each Splunk instance type (indexer, search head, or cluster master) from which you wish to pull metrics. At present, only one of each type is accepted, per configured receiver instance. This means, for example, that if you have three different "indexer" type instances that you would like to pull metrics from you will need to configure three different `splunkenterprise` receivers for each indexer node you wish to monitor. @@ -294,10 +293,10 @@ Each custom search requires: - `latest` (optional): The latest time for the search. Defaults to `now`. **Ignored if SPL already contains** `latest=`**.** - `metrics` (required): A list of metric definitions that map search result columns to OTel metrics. -**Note on time ranges:** +**Note on time ranges:** -- For regular searches (like `index=_internal | stats count`), the receiver automatically adds `earliest` and `latest` time modifiers. -- For generating commands that start with `|` (other than `| tstats`), no time range is added since these commands handle time differently. +- For regular searches (like `index=_internal | stats count`), the receiver automatically adds `earliest` and `latest` time modifiers. +- For generating commands that start with `|` (other than `| tstats`), no time range is added since these commands handle time differently. - Receiver will correctly add a time range to searches that **begin** with `| tstats` Each metric definition supports: @@ -406,5 +405,4 @@ receivers: search_source: "rest_api" ``` - For a full list of settings exposed by this receiver please look in [config.go](./config.go) with a detailed configuration in [testdata/config.yaml](./testdata/config.yaml). diff --git a/receiver/splunkhecreceiver/README.md b/receiver/splunkhecreceiver/README.md index 1e5cc1e507e4a..2152111a17fc6 100644 --- a/receiver/splunkhecreceiver/README.md +++ b/receiver/splunkhecreceiver/README.md @@ -8,7 +8,6 @@ The collector accepts data formatted as JSON [HEC events](https://help.splunk.co under any path or as EOL separated log [raw data](https://help.splunk.com/en/splunk-enterprise/get-data-in/get-started-with-getting-data-in/10.0/get-data-with-http-event-collector/format-events-for-http-event-collector#raw-event-parsing-0) if sent to the `raw_path` path. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics, logs | diff --git a/receiver/sqlqueryreceiver/README.md b/receiver/sqlqueryreceiver/README.md index 286553d34b889..4cee721f8b3e9 100644 --- a/receiver/sqlqueryreceiver/README.md +++ b/receiver/sqlqueryreceiver/README.md @@ -3,7 +3,6 @@ The SQL Query Receiver uses custom SQL queries to generate logs and/or metrics from a database connection. - | Status | | | ------------- |-----------| | Stability | [development]: logs | diff --git a/receiver/sqlserverreceiver/README.md b/receiver/sqlserverreceiver/README.md index 933e7aa281250..9170dd6582ebd 100644 --- a/receiver/sqlserverreceiver/README.md +++ b/receiver/sqlserverreceiver/README.md @@ -5,7 +5,6 @@ The `sqlserver` receiver grabs metrics/logs about a Microsoft SQL Server instanc Windows Performance Counters, or by directly connecting to the instance and querying it. Windows Performance Counters are only available when running on Windows. - | Status | | | ------------- |-----------| | Stability | [development]: logs | @@ -153,7 +152,7 @@ Top query collection enabled: ## Feature Gate -A new feature gate was added in `v0.129.0` for removing the `server.address` and `server.port` +A new feature gate was added in `v0.129.0` for removing the `server.address` and `server.port` resource attributes, as they are not identified as resources attributes in the semantic conventions. To enable it, pass the following argument to the Collector: @@ -165,8 +164,7 @@ To enable it, pass the following argument to the Collector: Details about the metrics produced by this receiver can be found in [documentation.md](./documentation.md) - -## Logs +## Logs Details about the logs produced by this receiver can be found in [logs-documentation.md](./logs-documentation.md) diff --git a/receiver/sqlserverreceiver/logs-documentation.md b/receiver/sqlserverreceiver/logs-documentation.md index 1c0413f7b3816..14f5e8d63dfc3 100644 --- a/receiver/sqlserverreceiver/logs-documentation.md +++ b/receiver/sqlserverreceiver/logs-documentation.md @@ -51,4 +51,3 @@ | sqlserver.query_plan_hash | Hash value used to group similar query execution plans. | string | | sqlserver.context_info | Optional context information for the query/session. | string | | sqlserver.username | Login name associated with the SQL Server session. | string | - diff --git a/receiver/sshcheckreceiver/README.md b/receiver/sshcheckreceiver/README.md index 9357e34215c17..e6a5de6505a36 100644 --- a/receiver/sshcheckreceiver/README.md +++ b/receiver/sshcheckreceiver/README.md @@ -212,7 +212,7 @@ For production use, always configure proper host key validation using `known_hos ## Feature Gates -This receiver does not currently use any feature gates. All functionality is available through configuration options. +This receiver does not currently use any feature gates. All functionality is available through configuration options. ## Metrics @@ -235,6 +235,3 @@ The receiver adds the following resource attribute: ### Detailed Metric Documentation Complete details about the metrics produced by this receiver, including attributes, types, and units, can be found in [documentation.md](./documentation.md) and [metadata.yaml](./metadata.yaml). - - - diff --git a/receiver/statsdreceiver/README.md b/receiver/statsdreceiver/README.md index 099c4f20ab340..23ad00c93b4dc 100644 --- a/receiver/statsdreceiver/README.md +++ b/receiver/statsdreceiver/README.md @@ -5,7 +5,6 @@ StatsD receiver for ingesting StatsD messages(https://github.com/statsd/statsd/b the OpenTelemetry Collector. Note: This receiver does not support horizontally scaled collector deployments. It is intended to run in agent mode, where a single collector instance receives StatsD input. - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -28,7 +27,7 @@ The Following settings are optional: - `transport` (default = `udp`): Protocol used by the StatsD server. Currently supported transports can be found in [this file](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/statsdreceiver/internal/transport/transport.go). -- `socket_permissions` (default = `0622`): When transport is set to `unixgram`, can be used to customize permissions of the binded socket. +- `socket_permissions` (default = `0622`): When transport is set to `unixgram`, can be used to customize permissions of the binded socket. - `aggregation_interval: 70s`(default value is 60s): The aggregation time that the receiver aggregates the metrics (similar to the flush interval in StatsD server) @@ -44,12 +43,11 @@ The Following settings are optional: - `timer_histogram_mapping:`(default value is below): Specify what OTLP type to convert received timing/histogram data to. - `"statsd_type"` specifies received Statsd data type. Possible values for this setting are `"timing"`, `"timer"`, `"histogram"` and `"distribution"`. `"observer_type"` specifies OTLP data type to convert to. We support `"gauge"`, `"summary"`, and `"histogram"`. For `"gauge"`, it does not perform any aggregation. For `"summary`, the statsD receiver will aggregate to one OTLP summary metric for one metric description (the same metric name with the same tags). By default, it will send percentile 0, 10, 50, 90, 95, 100 to the downstream. The `"histogram"` setting selects an [auto-scaling exponential histogram configured with only a maximum size](https://github.com/lightstep/go-expohisto#readme), as shown in the example below unless it matches the configured explicit_buckets matcher pattern. -TODO: Add a new option to use a smoothed summary like Prometheus: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/3261 +TODO: Add a new option to use a smoothed summary like Prometheus: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/3261 Example: @@ -125,12 +123,10 @@ It supports sample rate. When a sample rate is provided (between 0 and 1), the r **Note**: Currently, the sample rate is applied during aggregation and the adjusted value is exported. If OTLP adds native support for sample rate as a metric parameter in the future, the implementation may be updated to preserve the original sample rate information. - ### Gauge `:|g|@|#:` - ### Timer `:|ms|@|#:` @@ -138,7 +134,6 @@ It supports sample rate. When a sample rate is provided (between 0 and 1), the r It supports sample rate. - ## Testing ### Full sample collector config diff --git a/receiver/stefreceiver/README.md b/receiver/stefreceiver/README.md index 57dda71245481..2fbc2cfaa1ecf 100644 --- a/receiver/stefreceiver/README.md +++ b/receiver/stefreceiver/README.md @@ -3,7 +3,6 @@ Receives data via gRPC in OTel/STEF format. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/receiver/syslogreceiver/README.md b/receiver/syslogreceiver/README.md index a770c5e329420..8423bd8494088 100644 --- a/receiver/syslogreceiver/README.md +++ b/receiver/syslogreceiver/README.md @@ -155,4 +155,3 @@ receivers: protocol: rfc3164 location: UTC ``` - diff --git a/receiver/systemdreceiver/documentation.md b/receiver/systemdreceiver/documentation.md index 05e6b2ee4e3b6..468b4b991cb60 100644 --- a/receiver/systemdreceiver/documentation.md +++ b/receiver/systemdreceiver/documentation.md @@ -58,7 +58,6 @@ This exposes services' `NRestarts` property as a metric. This only tracks automatic service restarts (restarts when the process exits), and does not include manual restarts (e.g. from `systemctl restart`). - | Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | | ---- | ----------- | ---------- | ----------------------- | --------- | --------- | | {restarts} | Sum | Int | Cumulative | true | Development | diff --git a/receiver/tcpcheckreceiver/README.md b/receiver/tcpcheckreceiver/README.md index fd1a30419d4fe..a61e2238e2ecd 100644 --- a/receiver/tcpcheckreceiver/README.md +++ b/receiver/tcpcheckreceiver/README.md @@ -48,4 +48,3 @@ The full list of settings exposed for this receiver are documented [here](./conf ## Metrics Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) - diff --git a/receiver/tcplogreceiver/README.md b/receiver/tcplogreceiver/README.md index 7fa6a428c6f1e..ee6672973198d 100644 --- a/receiver/tcplogreceiver/README.md +++ b/receiver/tcplogreceiver/README.md @@ -26,7 +26,7 @@ The TCP Log Receiver receives logs over TCP. | `attributes` | {} | A map of `key: value` pairs to add to the entry's attributes | | `one_log_per_packet` | false | Skip log tokenization, set to true if logs contains one log per record and multiline is not used. This will improve performance. | | `resource` | {} | A map of `key: value` pairs to add to the entry's resource | -| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention][https://github.com/open-telemetry/semantic-conventions/blob/main/docs/registry/attributes/network.md#network-attributes] | +| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/registry/attributes/network.md#network-attributes) | | `multiline` | | A `multiline` configuration block. See below for details | | `encoding` | `utf-8` | The encoding of the file being read. See the list of supported encodings below for available options | | `operators` | [] | An array of [operators](../../pkg/stanza/docs/operators/README.md#what-operators-are-available). See below for more details | @@ -71,7 +71,7 @@ The `omit_pattern` setting can be used to omit the start/end pattern from each e #### Supported encodings -| Key | Description +| Key | Description | | --- | --- | | `nop` | No encoding validation. Treats the file as a stream of raw bytes | | `utf-8` | UTF-8 encoding | diff --git a/receiver/udplogreceiver/README.md b/receiver/udplogreceiver/README.md index 53adb18050aaf..165f1e64931ab 100644 --- a/receiver/udplogreceiver/README.md +++ b/receiver/udplogreceiver/README.md @@ -24,7 +24,7 @@ The UDP Log Receiver receives logs over UDP. | `attributes` | {} | A map of `key: value` pairs to add to the entry's attributes | | `one_log_per_packet` | false | Skip log tokenization, set to true if logs contains one log per record and multiline is not used. This will improve performance. | | `resource` | {} | A map of `key: value` pairs to add to the entry's resource | -| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention][https://github.com/open-telemetry/semantic-conventions/blob/cee22ec91448808ebcfa53df689c800c7171c9e1/docs/general/attributes.md#other-network-attributes] | +| `add_attributes` | false | Adds `net.*` attributes according to [semantic convention](https://github.com/open-telemetry/semantic-conventions/blob/cee22ec91448808ebcfa53df689c800c7171c9e1/docs/general/attributes.md#other-network-attributes) | | `multiline` | | A `multiline` configuration block. See below for details | | `encoding` | `utf-8` | The encoding of the file being read. See the list of supported encodings below for available options | | `operators` | [] | An array of [operators](../../pkg/stanza/docs/operators/README.md#what-operators-are-available). See below for more details | @@ -57,7 +57,7 @@ The `omit_pattern` setting can be used to omit the start/end pattern from each e ### Supported encodings -| Key | Description +| Key | Description | | --- | --- | | `nop` | No encoding validation. Treats the file as a stream of raw bytes | | `utf-8` | UTF-8 encoding | diff --git a/receiver/vcenterreceiver/README.md b/receiver/vcenterreceiver/README.md index 643219dc31f8e..b8673edfdaf8a 100644 --- a/receiver/vcenterreceiver/README.md +++ b/receiver/vcenterreceiver/README.md @@ -27,7 +27,6 @@ A “Read Only” user assigned to a vSphere with permissions to the vCenter ser ## Configuration - | Parameter | Default | Type | Notes | | ------------------- | ------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | endpoint | | String | Endpoint to the vCenter Server or ESXi host that has the sdk path enabled. Required. The expected format is `://`

i.e: `https://vcsa.hostname.localnet` | @@ -60,7 +59,7 @@ Details about the metrics produced by this receiver can be found in [metadata.ya **ALPHA**: `receiver.vcenter.resourcePoolMemoryUsageAttribute` -The feature gate `receiver.vcenter.resourcePoolMemoryUsageAttribute` once enabled will enable the memory usage type attribute for +The feature gate `receiver.vcenter.resourcePoolMemoryUsageAttribute` once enabled will enable the memory usage type attribute for the `vcenter.resource_pool.memory.usage` metric. This feature gate will eventually be enabled by default, and eventually the old implementation will be removed. It aims diff --git a/receiver/vcrreceiver/README.md b/receiver/vcrreceiver/README.md index 838ac8a9ee908..2f1e303bc8f84 100644 --- a/receiver/vcrreceiver/README.md +++ b/receiver/vcrreceiver/README.md @@ -5,7 +5,6 @@ The OpenTelemetry **Verbatim Capture & Replay (VCR) toolset** enables full-fidel telemetry data stored on disk. It replays previously captured telemetry while **preserving original event timestamps**, allowing you to simulate realistic traffic timing independent of when the data was captured. - | Status | | | ------------- |-----------| | Stability | [development]: traces, metrics, logs, profiles | diff --git a/receiver/wavefrontreceiver/README.md b/receiver/wavefrontreceiver/README.md index 0305074beb879..e791714115a7e 100644 --- a/receiver/wavefrontreceiver/README.md +++ b/receiver/wavefrontreceiver/README.md @@ -29,7 +29,6 @@ it to the collector metric format. See [https://docs.wavefront.com/wavefront_data_format.html#metrics-data-format-syntax.](https://docs.wavefront.com/wavefront_data_format.html#metrics-data-format-syntax) Each line received represents a Wavefront metric in the following format: - ``` [] source= [pointTags]``` > :information_source: The `wavefront` receiver is based on Carbon and binds to the @@ -66,4 +65,3 @@ receivers: The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml). - diff --git a/receiver/webhookeventreceiver/README.md b/receiver/webhookeventreceiver/README.md index b0e88e1cc8269..b568c2e7fd119 100644 --- a/receiver/webhookeventreceiver/README.md +++ b/receiver/webhookeventreceiver/README.md @@ -6,7 +6,6 @@ for any webhook style data source. It is designed to work alongside other pipeli [transform processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor) to allow the ingestion of generic events as logs into the opentelemetry collector. - | Status | | | ------------- |-----------| | Stability | [beta]: logs | @@ -27,7 +26,7 @@ The following settings are required: The following settings are optional: -* `path` (default: '/events'): Path where the receiver instance will accept events +* `path` (default: '/events'): Path where the receiver instance will accept events * `health_path` (default: '/health_check'): Path available for checking receiver status * `read_timeout` (default: '500ms'): Maximum wait time while attempting to read a received event * `write_timeout` (default: '500ms'): Maximum wait time while attempting to write a response @@ -44,7 +43,7 @@ The following settings are optional: If the setting is unconfigured or set to `false`, the receiver will create a single log record with the entire request body as the "body" of that record. -If the webhook body looks like the following, use `split_logs_at_newline: false`: +If the webhook body looks like the following, use `split_logs_at_newline: false`: ```yaml { @@ -66,7 +65,7 @@ a third line Three log records will be created from this example. The first two are JSON body objects and the third is just the string "a third line". -This receiver does not attempt to marshal the body into a structured format as it is received so it cannot make a more intelligent determination about where the split records. +This receiver does not attempt to marshal the body into a structured format as it is received so it cannot make a more intelligent determination about where the split records. ### Split logs at JSON boundary example @@ -98,4 +97,3 @@ receivers: ``` The full list of settings exposed for this receiver are documented in [config.go](./config.go) with a detailed sample configuration in [testdata/config.yaml](./testdata/config.yaml) - diff --git a/receiver/windowseventlogreceiver/README.md b/receiver/windowseventlogreceiver/README.md index 0bd8aa685141b..33e68e62b3038 100644 --- a/receiver/windowseventlogreceiver/README.md +++ b/receiver/windowseventlogreceiver/README.md @@ -4,7 +4,6 @@ This receiver tails and parses logs from windows event log API using the [opentelemetry-log-collection](https://github.com/open-telemetry/opentelemetry-log-collection) library. - | Status | | | ------------- |-----------| | Stability | [alpha]: logs | @@ -34,7 +33,7 @@ This receiver tails and parses logs from windows event log API using the | `operators` | [] | An array of [operators](https://github.com/open-telemetry/opentelemetry-log-collection/blob/main/docs/operators/README.md#what-operators-are-available). See below for more details | | `raw` | false | If false, the body of emitted log records will contain a structured representation of the event. Otherwise, the body will be the original XML string. | | `event_data_format` | `map` | Controls the structure of the `event_data` field when `raw` is false. `map` emits a flat map (named elements as direct keys, anonymous elements as `param1`, `param2`, etc.). `array` emits the legacy format with a nested `data` array of single-key maps. | -| `include_log_record_original` | false | If false, no additional attributes are added. If true, `log.record.original` is added to the attributes, which stores the original XML string according to the configured `suppress_rendering_info` (see below). +| `include_log_record_original` | false | If false, no additional attributes are added. If true, `log.record.original` is added to the attributes, which stores the original XML string according to the configured `suppress_rendering_info` (see below). | | `suppress_rendering_info` | false | If false, [additional syscalls](https://learn.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage#remarks) may be made to retrieve detailed information about the event. Otherwise, some unresolved values may be present in the event. | | `exclude_providers` | [] | One or more event log providers to exclude from processing. | | `storage` | none | The ID of a storage extension to be used to store bookmarks. Bookmarks allow the receiver to pick up where it left off in the case of a collector restart. If no storage extension is used, the receiver will manage bookmarks in memory only. | @@ -50,7 +49,6 @@ This receiver tails and parses logs from windows event log API using the | `resolve_sids.cache_ttl` | `15m` | Time-to-live for cached SID mappings. After this duration, SIDs will be re-resolved from the Windows LSA API. | | `discover_domain_controllers` | `false` | Automatically discover and collect events from Active Directory domain controllers. | - ### Feature Gates | Feature Gate | Stage | Description | @@ -175,7 +173,6 @@ If collection of the local event log is desired, a separate receiver needs to be - The remote computer must enable the "Remote Event Log Management" Windows Firewall exception. Otherwise, when you try to use the session handle, the call will error with `RPC_S_SERVER_UNAVAILABLE`. - The computer to which you are connecting must be running Windows Vista or later. - Single server configuration: ```yaml receivers: diff --git a/receiver/windowsperfcountersreceiver/README.md b/receiver/windowsperfcountersreceiver/README.md index 56fbb10f93268..09a8eaa8bea7f 100644 --- a/receiver/windowsperfcountersreceiver/README.md +++ b/receiver/windowsperfcountersreceiver/README.md @@ -5,7 +5,6 @@ This receiver, for Windows only, captures the configured system, application, or custom performance counter data from the Windows registry using the [PDH interface](https://docs.microsoft.com/en-us/windows/win32/perfctrs/using-the-pdh-functions-to-consume-counter-data). - | Status | | | ------------- |-----------| | Stability | [beta]: metrics | @@ -60,13 +59,13 @@ windowsperfcounters: ### Understanding the `instances` configuration option -Value | Interpretation --- | -- -Not specified | This is the only valid value if the counter has no instances. -`"*"` | All instances, excluding `_Total`. -`"_Total"` | The "total" instance, that aggregates the values of all other instances. See below for its special treatment. -`"instance1"` | A single instance. -`["instance1", "instance2", ...]` | A set of instances. +| Value | Interpretation | +| -- | -- | +| Not specified | This is the only valid value if the counter has no instances. | +| `"*"` | All instances, excluding `_Total`. | +| `"_Total"` | The "total" instance, that aggregates the values of all other instances. See below for its special treatment. | +| `"instance1"` | A single instance. | +| `["instance1", "instance2", ...]` | A set of instances. | ### Aggregation counter and the behavior of the `_Total` instance @@ -160,7 +159,7 @@ service: ### Defining metric format -To report metrics in the desired output format, define a metric and reference it in the corresponding counter, along with any applicable attributes. The metric's data type can either be `gauge` (default) or `sum`. +To report metrics in the desired output format, define a metric and reference it in the corresponding counter, along with any applicable attributes. The metric's data type can either be `gauge` (default) or `sum`. | Field Name | Description | Value | Default | | -- | -- | -- | -- | @@ -170,7 +169,6 @@ To report metrics in the desired output format, define a metric and reference it | sum | representation of a sum metric. | Sum Config | | | gauge | representation of a gauge metric. | Gauge Config | | - #### Sum Config | Field Name | Description | Value | Default | diff --git a/receiver/windowsservicereceiver/README.md b/receiver/windowsservicereceiver/README.md index 13632e48ba7a6..903edee9bcd1f 100644 --- a/receiver/windowsservicereceiver/README.md +++ b/receiver/windowsservicereceiver/README.md @@ -4,7 +4,6 @@ The Windows Service Receiver is a receiver for scraping information about the state of services running on a Windows machine. - | Status | | | ------------- |-----------| | Stability | [development]: metrics | diff --git a/receiver/windowsservicereceiver/documentation.md b/receiver/windowsservicereceiver/documentation.md index ec3d5c14bba85..b72c5183c0f00 100644 --- a/receiver/windowsservicereceiver/documentation.md +++ b/receiver/windowsservicereceiver/documentation.md @@ -18,7 +18,6 @@ Gauge value containing service status as an integer value. Gauge values map to service status as follows: 0 - Failed to retrieve service status, 1 - Stopped, 2 - Start Pending, 3 - Stop Pending, 4 - Service Running, 5 - Continue Pending, 6 - Pause Pending, 7 - Service Paused - | Unit | Metric Type | Value Type | Stability | | ---- | ----------- | ---------- | --------- | | {status} | Gauge | Int | Development | diff --git a/receiver/yanggrpcreceiver/README.md b/receiver/yanggrpcreceiver/README.md index 3e23309a31e5a..45803d372fba1 100644 --- a/receiver/yanggrpcreceiver/README.md +++ b/receiver/yanggrpcreceiver/README.md @@ -5,7 +5,6 @@ The YANG/gRPC receiver receives metrics offered using the [YANG (Yet Another New Generation) data model](https://en.wikipedia.org/wiki/YANG), expressed over [gRPC](https://ciscolearning.github.io/cisco-learning-codelabs/posts/yangsuite-restconf/#0). - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | @@ -90,7 +89,7 @@ yang_grpc: ## YANG Parser Settings (yang) * `enable_rfc_parser`: Enable RFC 6020/7950 compliant parsing. -* `cache_modules`: Local directories containing Cisco/IETF `.yang` files for accurate parsing. Cache discovered YANG modules to reduce CPU overhead. +* `cache_modules`: Local directories containing Cisco/IETF `.yang` files for accurate parsing. Cache discovered YANG modules to reduce CPU overhead. ## Production Deployment Example ```YAML diff --git a/receiver/zipkinreceiver/README.md b/receiver/zipkinreceiver/README.md index 7a5c61d385625..1d6e611d27f88 100644 --- a/receiver/zipkinreceiver/README.md +++ b/receiver/zipkinreceiver/README.md @@ -3,7 +3,6 @@ This receiver receives spans from [Zipkin](https://zipkin.io/) (V1 and V2). - | Status | | | ------------- |-----------| | Stability | [beta]: traces | diff --git a/receiver/zookeeperreceiver/README.md b/receiver/zookeeperreceiver/README.md index f1241efd1174d..90d2120bf386f 100644 --- a/receiver/zookeeperreceiver/README.md +++ b/receiver/zookeeperreceiver/README.md @@ -4,7 +4,6 @@ The Zookeeper Receiver collects metrics from a Zookeeper instance, using the `mntr` command. The `mntr` 4 letter word command needs to be enabled for the receiver to be able to collect metrics. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/scraper/zookeeperscraper/README.md b/scraper/zookeeperscraper/README.md index 2289843e93a6b..88d2f155a3515 100644 --- a/scraper/zookeeperscraper/README.md +++ b/scraper/zookeeperscraper/README.md @@ -4,7 +4,6 @@ The Zookeeper scraper collects metrics from a Zookeeper instance, using the `mntr` command. The `mntr` 4 letter word command needs to be enabled for the scraper to be able to collect metrics. - | Status | | | ------------- |-----------| | Stability | [alpha]: metrics | diff --git a/testbed/README.md b/testbed/README.md index 1be65a2e0f351..7bb9158cfd7f6 100644 --- a/testbed/README.md +++ b/testbed/README.md @@ -1,7 +1,7 @@ # OpenTelemetry Collector Testbed Testbed is a controlled environment and tools for conducting end-to-end tests for the Otel Collector, -including reproducible short-term benchmarks, correctness tests, long-running stability tests and +including reproducible short-term benchmarks, correctness tests, long-running stability tests and maximum load stress tests. ## Usage @@ -68,43 +68,43 @@ Generally, when designing a test for new exporter and receiver components, devel * ```go func TestTrace10kSPS(t *testing.T) { - tests := []struct { - name string - sender testbed.DataSender - receiver testbed.DataReceiver - resourceSpec testbed.ResourceSpec - }{ - { - "NewExporterOrReceiver", - testbed.NewXXXDataSender(testbed.DefaultHost, testutil.GetAvailablePort(t)), - testbed.NewXXXDataReceiver(testutil.GetAvailablePort(t)), - testbed.ResourceSpec{ - ExpectedMaxCPU: XX, - ExpectedMaxRAM: XX, - }, - }, - ... - } - processors := []ProcessorNameAndConfigBody{ - { - Name: "batch", - Body: ` - batch: + tests := []struct { + name string + sender testbed.DataSender + receiver testbed.DataReceiver + resourceSpec testbed.ResourceSpec + }{ + { + "NewExporterOrReceiver", + testbed.NewXXXDataSender(testbed.DefaultHost, testutil.GetAvailablePort(t)), + testbed.NewXXXDataReceiver(testutil.GetAvailablePort(t)), + testbed.ResourceSpec{ + ExpectedMaxCPU: XX, + ExpectedMaxRAM: XX, + }, + }, + ... + } + processors := []ProcessorNameAndConfigBody{ + { + Name: "batch", + Body: ` + batch: `, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - Scenario10kItemsPerSecond( - t, - test.sender, - test.receiver, - test.resourceSpec, - performanceResultsSummary, - processors, - ) - }) - } + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + Scenario10kItemsPerSecond( + t, + test.sender, + test.receiver, + test.resourceSpec, + performanceResultsSummary, + processors, + ) + }) + } } ``` @@ -131,4 +131,3 @@ Run the following at the root of the repo: ### Advanced usage A Makefile is also located at [`testbed/Makefile`](./Makefile) that offers targets to directly run certain test suites. Note that these targets will not compile the Collector before running. - From bb36dcccfc0228a20de26800fc3793b68d2454e2 Mon Sep 17 00:00:00 2001 From: Ravishankar Gnanaprakasam Date: Fri, 15 May 2026 15:19:33 +0530 Subject: [PATCH 2/2] [ci-cd] Fix link checks --- issue-triaging.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/issue-triaging.md b/issue-triaging.md index 74ebe0912aab0..0be2ceedd76ce 100644 --- a/issue-triaging.md +++ b/issue-triaging.md @@ -10,7 +10,7 @@ OpenTelemetry community members, issue authors, and anyone else who would like t #### Triagers -Contributors with [triager](/#Contributing) permissions can help move +Contributors with [triager](README.md#contributing) permissions can help move issues along by adding missing component labels, which help organize issues and trigger automations to notify code owners. They can also use their familiarity with the Collector and its components to investigate issues themselves. Alternatively, they may point issue authors to another resource or someone else who may know more. @@ -26,7 +26,7 @@ is fit to be added to a component. Code owners will be notified by repository au Code owners may not have triager permissions on the repository, so they can help triage through investigation and by participating in discussions. They can also help organize issues by -[adding labels via comments](/CONTRIBUTING.md#adding-labels-via-comments). +[adding labels via comments](CONTRIBUTING.md#adding-labels-via-comments). #### Community Members