Add opentelemetry integration to Oban (#6)
By default a new trace is automatically started when a job is processed
by monitoring these events:
* `[:oban, :job, :start]` — at the point a job is fetched from the database and will execute
* `[:oban, :job, :stop]` — after a job succeeds and the success is recorded in the database
* `[:oban, :job, :exception]` — after a job fails and the failure is recorded in the database
To also record a span when a job is created and to link traces together
`Oban.insert/2` has to be replaced by `OpentelemetryOban.insert/2`.
Before:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> Oban.insert()
```
After:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> OpentelemetryOban.insert()
```
Co-authored-by: Tristan Sloughter <t@crashfast.com>
2021-12-08 15:41:36 +00:00
|
|
|
defmodule OpentelemetryOban.JobHandler do
|
|
|
|
alias OpenTelemetry.Span
|
2022-12-14 23:48:27 +00:00
|
|
|
alias OpenTelemetry.SemanticConventions.Trace
|
|
|
|
|
|
|
|
require Trace
|
Add opentelemetry integration to Oban (#6)
By default a new trace is automatically started when a job is processed
by monitoring these events:
* `[:oban, :job, :start]` — at the point a job is fetched from the database and will execute
* `[:oban, :job, :stop]` — after a job succeeds and the success is recorded in the database
* `[:oban, :job, :exception]` — after a job fails and the failure is recorded in the database
To also record a span when a job is created and to link traces together
`Oban.insert/2` has to be replaced by `OpentelemetryOban.insert/2`.
Before:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> Oban.insert()
```
After:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> OpentelemetryOban.insert()
```
Co-authored-by: Tristan Sloughter <t@crashfast.com>
2021-12-08 15:41:36 +00:00
|
|
|
|
2021-12-28 23:39:06 +00:00
|
|
|
@tracer_id __MODULE__
|
Add opentelemetry integration to Oban (#6)
By default a new trace is automatically started when a job is processed
by monitoring these events:
* `[:oban, :job, :start]` — at the point a job is fetched from the database and will execute
* `[:oban, :job, :stop]` — after a job succeeds and the success is recorded in the database
* `[:oban, :job, :exception]` — after a job fails and the failure is recorded in the database
To also record a span when a job is created and to link traces together
`Oban.insert/2` has to be replaced by `OpentelemetryOban.insert/2`.
Before:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> Oban.insert()
```
After:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> OpentelemetryOban.insert()
```
Co-authored-by: Tristan Sloughter <t@crashfast.com>
2021-12-08 15:41:36 +00:00
|
|
|
|
|
|
|
def attach() do
|
|
|
|
attach_job_start_handler()
|
|
|
|
attach_job_stop_handler()
|
|
|
|
attach_job_exception_handler()
|
|
|
|
end
|
|
|
|
|
|
|
|
defp attach_job_start_handler() do
|
|
|
|
:telemetry.attach(
|
|
|
|
"#{__MODULE__}.job_start",
|
|
|
|
[:oban, :job, :start],
|
|
|
|
&__MODULE__.handle_job_start/4,
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
defp attach_job_stop_handler() do
|
|
|
|
:telemetry.attach(
|
|
|
|
"#{__MODULE__}.job_stop",
|
|
|
|
[:oban, :job, :stop],
|
|
|
|
&__MODULE__.handle_job_stop/4,
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
defp attach_job_exception_handler() do
|
|
|
|
:telemetry.attach(
|
|
|
|
"#{__MODULE__}.job_exception",
|
|
|
|
[:oban, :job, :exception],
|
|
|
|
&__MODULE__.handle_job_exception/4,
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
def handle_job_start(_event, _measurements, metadata, _config) do
|
|
|
|
%{
|
|
|
|
job: %{
|
|
|
|
id: id,
|
|
|
|
queue: queue,
|
|
|
|
worker: worker,
|
|
|
|
priority: priority,
|
|
|
|
inserted_at: inserted_at,
|
|
|
|
scheduled_at: scheduled_at,
|
|
|
|
attempt: attempt,
|
|
|
|
max_attempts: max_attempts,
|
|
|
|
meta: job_meta
|
|
|
|
}
|
|
|
|
} = metadata
|
|
|
|
|
|
|
|
:otel_propagator_text_map.extract(Map.to_list(job_meta))
|
|
|
|
parent = OpenTelemetry.Tracer.current_span_ctx()
|
|
|
|
links = if parent == :undefined, do: [], else: [OpenTelemetry.link(parent)]
|
|
|
|
OpenTelemetry.Tracer.set_current_span(:undefined)
|
|
|
|
|
2021-12-28 23:39:06 +00:00
|
|
|
attributes = %{
|
2022-12-14 23:48:27 +00:00
|
|
|
Trace.messaging_system() => :oban,
|
|
|
|
Trace.messaging_destination() => queue,
|
|
|
|
Trace.messaging_destination_kind() => :queue,
|
|
|
|
Trace.messaging_operation() => :process,
|
|
|
|
:"messaging.oban.job_id" => id,
|
|
|
|
:"messaging.oban.worker" => worker,
|
|
|
|
:"messaging.oban.priority" => priority,
|
|
|
|
:"messaging.oban.attempt" => attempt,
|
|
|
|
:"messaging.oban.max_attempts" => max_attempts,
|
|
|
|
:"messaging.oban.inserted_at" =>
|
Add opentelemetry integration to Oban (#6)
By default a new trace is automatically started when a job is processed
by monitoring these events:
* `[:oban, :job, :start]` — at the point a job is fetched from the database and will execute
* `[:oban, :job, :stop]` — after a job succeeds and the success is recorded in the database
* `[:oban, :job, :exception]` — after a job fails and the failure is recorded in the database
To also record a span when a job is created and to link traces together
`Oban.insert/2` has to be replaced by `OpentelemetryOban.insert/2`.
Before:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> Oban.insert()
```
After:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> OpentelemetryOban.insert()
```
Co-authored-by: Tristan Sloughter <t@crashfast.com>
2021-12-08 15:41:36 +00:00
|
|
|
if(inserted_at, do: DateTime.to_iso8601(inserted_at), else: nil),
|
2022-12-14 23:48:27 +00:00
|
|
|
:"messaging.oban.scheduled_at" => DateTime.to_iso8601(scheduled_at)
|
2021-12-28 23:39:06 +00:00
|
|
|
}
|
Add opentelemetry integration to Oban (#6)
By default a new trace is automatically started when a job is processed
by monitoring these events:
* `[:oban, :job, :start]` — at the point a job is fetched from the database and will execute
* `[:oban, :job, :stop]` — after a job succeeds and the success is recorded in the database
* `[:oban, :job, :exception]` — after a job fails and the failure is recorded in the database
To also record a span when a job is created and to link traces together
`Oban.insert/2` has to be replaced by `OpentelemetryOban.insert/2`.
Before:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> Oban.insert()
```
After:
```elixir
%{id: 1, in_the: "business", of_doing: "business"}
|> MyApp.Business.new()
|> OpentelemetryOban.insert()
```
Co-authored-by: Tristan Sloughter <t@crashfast.com>
2021-12-08 15:41:36 +00:00
|
|
|
|
|
|
|
span_name = "#{worker} process"
|
|
|
|
|
|
|
|
OpentelemetryTelemetry.start_telemetry_span(@tracer_id, span_name, metadata, %{
|
|
|
|
kind: :consumer,
|
|
|
|
links: links,
|
|
|
|
attributes: attributes
|
|
|
|
})
|
|
|
|
end
|
|
|
|
|
|
|
|
def handle_job_stop(_event, _measurements, metadata, _config) do
|
|
|
|
OpentelemetryTelemetry.end_telemetry_span(@tracer_id, metadata)
|
|
|
|
end
|
|
|
|
|
|
|
|
def handle_job_exception(
|
|
|
|
_event,
|
|
|
|
_measurements,
|
|
|
|
%{stacktrace: stacktrace, error: error} = metadata,
|
|
|
|
_config
|
|
|
|
) do
|
|
|
|
ctx = OpentelemetryTelemetry.set_current_telemetry_span(@tracer_id, metadata)
|
|
|
|
|
|
|
|
# Record exception and mark the span as errored
|
|
|
|
Span.record_exception(ctx, error, stacktrace)
|
|
|
|
Span.set_status(ctx, OpenTelemetry.status(:error, ""))
|
|
|
|
|
|
|
|
OpentelemetryTelemetry.end_telemetry_span(@tracer_id, metadata)
|
|
|
|
end
|
|
|
|
end
|