rq.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161
  1. import weakref
  2. import sentry_sdk
  3. from sentry_sdk.consts import OP
  4. from sentry_sdk.api import continue_trace
  5. from sentry_sdk.integrations import _check_minimum_version, DidNotEnable, Integration
  6. from sentry_sdk.integrations.logging import ignore_logger
  7. from sentry_sdk.tracing import TransactionSource
  8. from sentry_sdk.utils import (
  9. capture_internal_exceptions,
  10. ensure_integration_enabled,
  11. event_from_exception,
  12. format_timestamp,
  13. parse_version,
  14. )
  15. try:
  16. from rq.queue import Queue
  17. from rq.timeouts import JobTimeoutException
  18. from rq.version import VERSION as RQ_VERSION
  19. from rq.worker import Worker
  20. from rq.job import JobStatus
  21. except ImportError:
  22. raise DidNotEnable("RQ not installed")
  23. from typing import TYPE_CHECKING
  24. if TYPE_CHECKING:
  25. from typing import Any, Callable
  26. from sentry_sdk._types import Event, EventProcessor
  27. from sentry_sdk.utils import ExcInfo
  28. from rq.job import Job
  29. class RqIntegration(Integration):
  30. identifier = "rq"
  31. origin = f"auto.queue.{identifier}"
  32. @staticmethod
  33. def setup_once():
  34. # type: () -> None
  35. version = parse_version(RQ_VERSION)
  36. _check_minimum_version(RqIntegration, version)
  37. old_perform_job = Worker.perform_job
  38. @ensure_integration_enabled(RqIntegration, old_perform_job)
  39. def sentry_patched_perform_job(self, job, *args, **kwargs):
  40. # type: (Any, Job, *Queue, **Any) -> bool
  41. with sentry_sdk.new_scope() as scope:
  42. scope.clear_breadcrumbs()
  43. scope.add_event_processor(_make_event_processor(weakref.ref(job)))
  44. transaction = continue_trace(
  45. job.meta.get("_sentry_trace_headers") or {},
  46. op=OP.QUEUE_TASK_RQ,
  47. name="unknown RQ task",
  48. source=TransactionSource.TASK,
  49. origin=RqIntegration.origin,
  50. )
  51. with capture_internal_exceptions():
  52. transaction.name = job.func_name
  53. with sentry_sdk.start_transaction(
  54. transaction,
  55. custom_sampling_context={"rq_job": job},
  56. ):
  57. rv = old_perform_job(self, job, *args, **kwargs)
  58. if self.is_horse:
  59. # We're inside of a forked process and RQ is
  60. # about to call `os._exit`. Make sure that our
  61. # events get sent out.
  62. sentry_sdk.get_client().flush()
  63. return rv
  64. Worker.perform_job = sentry_patched_perform_job
  65. old_handle_exception = Worker.handle_exception
  66. def sentry_patched_handle_exception(self, job, *exc_info, **kwargs):
  67. # type: (Worker, Any, *Any, **Any) -> Any
  68. retry = (
  69. hasattr(job, "retries_left")
  70. and job.retries_left
  71. and job.retries_left > 0
  72. )
  73. failed = job._status == JobStatus.FAILED or job.is_failed
  74. if failed and not retry:
  75. _capture_exception(exc_info)
  76. return old_handle_exception(self, job, *exc_info, **kwargs)
  77. Worker.handle_exception = sentry_patched_handle_exception
  78. old_enqueue_job = Queue.enqueue_job
  79. @ensure_integration_enabled(RqIntegration, old_enqueue_job)
  80. def sentry_patched_enqueue_job(self, job, **kwargs):
  81. # type: (Queue, Any, **Any) -> Any
  82. scope = sentry_sdk.get_current_scope()
  83. if scope.span is not None:
  84. job.meta["_sentry_trace_headers"] = dict(
  85. scope.iter_trace_propagation_headers()
  86. )
  87. return old_enqueue_job(self, job, **kwargs)
  88. Queue.enqueue_job = sentry_patched_enqueue_job
  89. ignore_logger("rq.worker")
  90. def _make_event_processor(weak_job):
  91. # type: (Callable[[], Job]) -> EventProcessor
  92. def event_processor(event, hint):
  93. # type: (Event, dict[str, Any]) -> Event
  94. job = weak_job()
  95. if job is not None:
  96. with capture_internal_exceptions():
  97. extra = event.setdefault("extra", {})
  98. rq_job = {
  99. "job_id": job.id,
  100. "func": job.func_name,
  101. "args": job.args,
  102. "kwargs": job.kwargs,
  103. "description": job.description,
  104. }
  105. if job.enqueued_at:
  106. rq_job["enqueued_at"] = format_timestamp(job.enqueued_at)
  107. if job.started_at:
  108. rq_job["started_at"] = format_timestamp(job.started_at)
  109. extra["rq-job"] = rq_job
  110. if "exc_info" in hint:
  111. with capture_internal_exceptions():
  112. if issubclass(hint["exc_info"][0], JobTimeoutException):
  113. event["fingerprint"] = ["rq", "JobTimeoutException", job.func_name]
  114. return event
  115. return event_processor
  116. def _capture_exception(exc_info, **kwargs):
  117. # type: (ExcInfo, **Any) -> None
  118. client = sentry_sdk.get_client()
  119. event, hint = event_from_exception(
  120. exc_info,
  121. client_options=client.options,
  122. mechanism={"type": "rq", "handled": False},
  123. )
  124. sentry_sdk.capture_event(event, hint=hint)