tracing_utils.py 38 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196
  1. import contextlib
  2. import functools
  3. import inspect
  4. import os
  5. import re
  6. import sys
  7. from collections.abc import Mapping
  8. from datetime import timedelta
  9. from random import Random
  10. from urllib.parse import quote, unquote
  11. import uuid
  12. import sentry_sdk
  13. from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS, SPANTEMPLATE
  14. from sentry_sdk.utils import (
  15. capture_internal_exceptions,
  16. filename_for_module,
  17. Dsn,
  18. logger,
  19. match_regex_list,
  20. qualname_from_function,
  21. safe_repr,
  22. to_string,
  23. try_convert,
  24. is_sentry_url,
  25. _is_external_source,
  26. _is_in_project_root,
  27. _module_in_list,
  28. )
  29. from typing import TYPE_CHECKING
  30. if TYPE_CHECKING:
  31. from typing import Any
  32. from typing import Dict
  33. from typing import Generator
  34. from typing import Optional
  35. from typing import Union
  36. from types import FrameType
  37. SENTRY_TRACE_REGEX = re.compile(
  38. "^[ \t]*" # whitespace
  39. "([0-9a-f]{32})?" # trace_id
  40. "-?([0-9a-f]{16})?" # span_id
  41. "-?([01])?" # sampled
  42. "[ \t]*$" # whitespace
  43. )
  44. # This is a normal base64 regex, modified to reflect that fact that we strip the
  45. # trailing = or == off
  46. base64_stripped = (
  47. # any of the characters in the base64 "alphabet", in multiples of 4
  48. "([a-zA-Z0-9+/]{4})*"
  49. # either nothing or 2 or 3 base64-alphabet characters (see
  50. # https://en.wikipedia.org/wiki/Base64#Decoding_Base64_without_padding for
  51. # why there's never only 1 extra character)
  52. "([a-zA-Z0-9+/]{2,3})?"
  53. )
  54. class EnvironHeaders(Mapping): # type: ignore
  55. def __init__(
  56. self,
  57. environ, # type: Mapping[str, str]
  58. prefix="HTTP_", # type: str
  59. ):
  60. # type: (...) -> None
  61. self.environ = environ
  62. self.prefix = prefix
  63. def __getitem__(self, key):
  64. # type: (str) -> Optional[Any]
  65. return self.environ[self.prefix + key.replace("-", "_").upper()]
  66. def __len__(self):
  67. # type: () -> int
  68. return sum(1 for _ in iter(self))
  69. def __iter__(self):
  70. # type: () -> Generator[str, None, None]
  71. for k in self.environ:
  72. if not isinstance(k, str):
  73. continue
  74. k = k.replace("-", "_").upper()
  75. if not k.startswith(self.prefix):
  76. continue
  77. yield k[len(self.prefix) :]
  78. def has_tracing_enabled(options):
  79. # type: (Optional[Dict[str, Any]]) -> bool
  80. """
  81. Returns True if either traces_sample_rate or traces_sampler is
  82. defined and enable_tracing is set and not false.
  83. """
  84. if options is None:
  85. return False
  86. return bool(
  87. options.get("enable_tracing") is not False
  88. and (
  89. options.get("traces_sample_rate") is not None
  90. or options.get("traces_sampler") is not None
  91. )
  92. )
  93. @contextlib.contextmanager
  94. def record_sql_queries(
  95. cursor, # type: Any
  96. query, # type: Any
  97. params_list, # type: Any
  98. paramstyle, # type: Optional[str]
  99. executemany, # type: bool
  100. record_cursor_repr=False, # type: bool
  101. span_origin="manual", # type: str
  102. ):
  103. # type: (...) -> Generator[sentry_sdk.tracing.Span, None, None]
  104. # TODO: Bring back capturing of params by default
  105. if sentry_sdk.get_client().options["_experiments"].get("record_sql_params", False):
  106. if not params_list or params_list == [None]:
  107. params_list = None
  108. if paramstyle == "pyformat":
  109. paramstyle = "format"
  110. else:
  111. params_list = None
  112. paramstyle = None
  113. query = _format_sql(cursor, query)
  114. data = {}
  115. if params_list is not None:
  116. data["db.params"] = params_list
  117. if paramstyle is not None:
  118. data["db.paramstyle"] = paramstyle
  119. if executemany:
  120. data["db.executemany"] = True
  121. if record_cursor_repr and cursor is not None:
  122. data["db.cursor"] = cursor
  123. with capture_internal_exceptions():
  124. sentry_sdk.add_breadcrumb(message=query, category="query", data=data)
  125. with sentry_sdk.start_span(
  126. op=OP.DB,
  127. name=query,
  128. origin=span_origin,
  129. ) as span:
  130. for k, v in data.items():
  131. span.set_data(k, v)
  132. yield span
  133. def maybe_create_breadcrumbs_from_span(scope, span):
  134. # type: (sentry_sdk.Scope, sentry_sdk.tracing.Span) -> None
  135. if span.op == OP.DB_REDIS:
  136. scope.add_breadcrumb(
  137. message=span.description, type="redis", category="redis", data=span._tags
  138. )
  139. elif span.op == OP.HTTP_CLIENT:
  140. level = None
  141. status_code = span._data.get(SPANDATA.HTTP_STATUS_CODE)
  142. if status_code:
  143. if 500 <= status_code <= 599:
  144. level = "error"
  145. elif 400 <= status_code <= 499:
  146. level = "warning"
  147. if level:
  148. scope.add_breadcrumb(
  149. type="http", category="httplib", data=span._data, level=level
  150. )
  151. else:
  152. scope.add_breadcrumb(type="http", category="httplib", data=span._data)
  153. elif span.op == "subprocess":
  154. scope.add_breadcrumb(
  155. type="subprocess",
  156. category="subprocess",
  157. message=span.description,
  158. data=span._data,
  159. )
  160. def _get_frame_module_abs_path(frame):
  161. # type: (FrameType) -> Optional[str]
  162. try:
  163. return frame.f_code.co_filename
  164. except Exception:
  165. return None
  166. def _should_be_included(
  167. is_sentry_sdk_frame, # type: bool
  168. namespace, # type: Optional[str]
  169. in_app_include, # type: Optional[list[str]]
  170. in_app_exclude, # type: Optional[list[str]]
  171. abs_path, # type: Optional[str]
  172. project_root, # type: Optional[str]
  173. ):
  174. # type: (...) -> bool
  175. # in_app_include takes precedence over in_app_exclude
  176. should_be_included = _module_in_list(namespace, in_app_include)
  177. should_be_excluded = _is_external_source(abs_path) or _module_in_list(
  178. namespace, in_app_exclude
  179. )
  180. return not is_sentry_sdk_frame and (
  181. should_be_included
  182. or (_is_in_project_root(abs_path, project_root) and not should_be_excluded)
  183. )
  184. def add_query_source(span):
  185. # type: (sentry_sdk.tracing.Span) -> None
  186. """
  187. Adds OTel compatible source code information to the span
  188. """
  189. client = sentry_sdk.get_client()
  190. if not client.is_active():
  191. return
  192. if span.timestamp is None or span.start_timestamp is None:
  193. return
  194. should_add_query_source = client.options.get("enable_db_query_source", True)
  195. if not should_add_query_source:
  196. return
  197. duration = span.timestamp - span.start_timestamp
  198. threshold = client.options.get("db_query_source_threshold_ms", 0)
  199. slow_query = duration / timedelta(milliseconds=1) > threshold
  200. if not slow_query:
  201. return
  202. project_root = client.options["project_root"]
  203. in_app_include = client.options.get("in_app_include")
  204. in_app_exclude = client.options.get("in_app_exclude")
  205. # Find the correct frame
  206. frame = sys._getframe() # type: Union[FrameType, None]
  207. while frame is not None:
  208. abs_path = _get_frame_module_abs_path(frame)
  209. try:
  210. namespace = frame.f_globals.get("__name__") # type: Optional[str]
  211. except Exception:
  212. namespace = None
  213. is_sentry_sdk_frame = namespace is not None and namespace.startswith(
  214. "sentry_sdk."
  215. )
  216. should_be_included = _should_be_included(
  217. is_sentry_sdk_frame=is_sentry_sdk_frame,
  218. namespace=namespace,
  219. in_app_include=in_app_include,
  220. in_app_exclude=in_app_exclude,
  221. abs_path=abs_path,
  222. project_root=project_root,
  223. )
  224. if should_be_included:
  225. break
  226. frame = frame.f_back
  227. else:
  228. frame = None
  229. # Set the data
  230. if frame is not None:
  231. try:
  232. lineno = frame.f_lineno
  233. except Exception:
  234. lineno = None
  235. if lineno is not None:
  236. span.set_data(SPANDATA.CODE_LINENO, frame.f_lineno)
  237. try:
  238. namespace = frame.f_globals.get("__name__")
  239. except Exception:
  240. namespace = None
  241. if namespace is not None:
  242. span.set_data(SPANDATA.CODE_NAMESPACE, namespace)
  243. filepath = _get_frame_module_abs_path(frame)
  244. if filepath is not None:
  245. if namespace is not None:
  246. in_app_path = filename_for_module(namespace, filepath)
  247. elif project_root is not None and filepath.startswith(project_root):
  248. in_app_path = filepath.replace(project_root, "").lstrip(os.sep)
  249. else:
  250. in_app_path = filepath
  251. span.set_data(SPANDATA.CODE_FILEPATH, in_app_path)
  252. try:
  253. code_function = frame.f_code.co_name
  254. except Exception:
  255. code_function = None
  256. if code_function is not None:
  257. span.set_data(SPANDATA.CODE_FUNCTION, frame.f_code.co_name)
  258. def extract_sentrytrace_data(header):
  259. # type: (Optional[str]) -> Optional[Dict[str, Union[str, bool, None]]]
  260. """
  261. Given a `sentry-trace` header string, return a dictionary of data.
  262. """
  263. if not header:
  264. return None
  265. if header.startswith("00-") and header.endswith("-00"):
  266. header = header[3:-3]
  267. match = SENTRY_TRACE_REGEX.match(header)
  268. if not match:
  269. return None
  270. trace_id, parent_span_id, sampled_str = match.groups()
  271. parent_sampled = None
  272. if trace_id:
  273. trace_id = "{:032x}".format(int(trace_id, 16))
  274. if parent_span_id:
  275. parent_span_id = "{:016x}".format(int(parent_span_id, 16))
  276. if sampled_str:
  277. parent_sampled = sampled_str != "0"
  278. return {
  279. "trace_id": trace_id,
  280. "parent_span_id": parent_span_id,
  281. "parent_sampled": parent_sampled,
  282. }
  283. def _format_sql(cursor, sql):
  284. # type: (Any, str) -> Optional[str]
  285. real_sql = None
  286. # If we're using psycopg2, it could be that we're
  287. # looking at a query that uses Composed objects. Use psycopg2's mogrify
  288. # function to format the query. We lose per-parameter trimming but gain
  289. # accuracy in formatting.
  290. try:
  291. if hasattr(cursor, "mogrify"):
  292. real_sql = cursor.mogrify(sql)
  293. if isinstance(real_sql, bytes):
  294. real_sql = real_sql.decode(cursor.connection.encoding)
  295. except Exception:
  296. real_sql = None
  297. return real_sql or to_string(sql)
  298. class PropagationContext:
  299. """
  300. The PropagationContext represents the data of a trace in Sentry.
  301. """
  302. __slots__ = (
  303. "_trace_id",
  304. "_span_id",
  305. "parent_span_id",
  306. "parent_sampled",
  307. "dynamic_sampling_context",
  308. )
  309. def __init__(
  310. self,
  311. trace_id=None, # type: Optional[str]
  312. span_id=None, # type: Optional[str]
  313. parent_span_id=None, # type: Optional[str]
  314. parent_sampled=None, # type: Optional[bool]
  315. dynamic_sampling_context=None, # type: Optional[Dict[str, str]]
  316. ):
  317. # type: (...) -> None
  318. self._trace_id = trace_id
  319. """The trace id of the Sentry trace."""
  320. self._span_id = span_id
  321. """The span id of the currently executing span."""
  322. self.parent_span_id = parent_span_id
  323. """The id of the parent span that started this span.
  324. The parent span could also be a span in an upstream service."""
  325. self.parent_sampled = parent_sampled
  326. """Boolean indicator if the parent span was sampled.
  327. Important when the parent span originated in an upstream service,
  328. because we want to sample the whole trace, or nothing from the trace."""
  329. self.dynamic_sampling_context = dynamic_sampling_context
  330. """Data that is used for dynamic sampling decisions."""
  331. @classmethod
  332. def from_incoming_data(cls, incoming_data):
  333. # type: (Dict[str, Any]) -> Optional[PropagationContext]
  334. propagation_context = None
  335. normalized_data = normalize_incoming_data(incoming_data)
  336. baggage_header = normalized_data.get(BAGGAGE_HEADER_NAME)
  337. if baggage_header:
  338. propagation_context = PropagationContext()
  339. propagation_context.dynamic_sampling_context = Baggage.from_incoming_header(
  340. baggage_header
  341. ).dynamic_sampling_context()
  342. sentry_trace_header = normalized_data.get(SENTRY_TRACE_HEADER_NAME)
  343. if sentry_trace_header:
  344. sentrytrace_data = extract_sentrytrace_data(sentry_trace_header)
  345. if sentrytrace_data is not None:
  346. if propagation_context is None:
  347. propagation_context = PropagationContext()
  348. propagation_context.update(sentrytrace_data)
  349. if propagation_context is not None:
  350. propagation_context._fill_sample_rand()
  351. return propagation_context
  352. @property
  353. def trace_id(self):
  354. # type: () -> str
  355. """The trace id of the Sentry trace."""
  356. if not self._trace_id:
  357. # New trace, don't fill in sample_rand
  358. self._trace_id = uuid.uuid4().hex
  359. return self._trace_id
  360. @trace_id.setter
  361. def trace_id(self, value):
  362. # type: (str) -> None
  363. self._trace_id = value
  364. @property
  365. def span_id(self):
  366. # type: () -> str
  367. """The span id of the currently executed span."""
  368. if not self._span_id:
  369. self._span_id = uuid.uuid4().hex[16:]
  370. return self._span_id
  371. @span_id.setter
  372. def span_id(self, value):
  373. # type: (str) -> None
  374. self._span_id = value
  375. def update(self, other_dict):
  376. # type: (Dict[str, Any]) -> None
  377. """
  378. Updates the PropagationContext with data from the given dictionary.
  379. """
  380. for key, value in other_dict.items():
  381. try:
  382. setattr(self, key, value)
  383. except AttributeError:
  384. pass
  385. def __repr__(self):
  386. # type: (...) -> str
  387. return "<PropagationContext _trace_id={} _span_id={} parent_span_id={} parent_sampled={} dynamic_sampling_context={}>".format(
  388. self._trace_id,
  389. self._span_id,
  390. self.parent_span_id,
  391. self.parent_sampled,
  392. self.dynamic_sampling_context,
  393. )
  394. def _fill_sample_rand(self):
  395. # type: () -> None
  396. """
  397. Ensure that there is a valid sample_rand value in the dynamic_sampling_context.
  398. If there is a valid sample_rand value in the dynamic_sampling_context, we keep it.
  399. Otherwise, we generate a sample_rand value according to the following:
  400. - If we have a parent_sampled value and a sample_rate in the DSC, we compute
  401. a sample_rand value randomly in the range:
  402. - [0, sample_rate) if parent_sampled is True,
  403. - or, in the range [sample_rate, 1) if parent_sampled is False.
  404. - If either parent_sampled or sample_rate is missing, we generate a random
  405. value in the range [0, 1).
  406. The sample_rand is deterministically generated from the trace_id, if present.
  407. This function does nothing if there is no dynamic_sampling_context.
  408. """
  409. if self.dynamic_sampling_context is None:
  410. return
  411. sample_rand = try_convert(
  412. float, self.dynamic_sampling_context.get("sample_rand")
  413. )
  414. if sample_rand is not None and 0 <= sample_rand < 1:
  415. # sample_rand is present and valid, so don't overwrite it
  416. return
  417. # Get the sample rate and compute the transformation that will map the random value
  418. # to the desired range: [0, 1), [0, sample_rate), or [sample_rate, 1).
  419. sample_rate = try_convert(
  420. float, self.dynamic_sampling_context.get("sample_rate")
  421. )
  422. lower, upper = _sample_rand_range(self.parent_sampled, sample_rate)
  423. try:
  424. sample_rand = _generate_sample_rand(self.trace_id, interval=(lower, upper))
  425. except ValueError:
  426. # ValueError is raised if the interval is invalid, i.e. lower >= upper.
  427. # lower >= upper might happen if the incoming trace's sampled flag
  428. # and sample_rate are inconsistent, e.g. sample_rate=0.0 but sampled=True.
  429. # We cannot generate a sensible sample_rand value in this case.
  430. logger.debug(
  431. f"Could not backfill sample_rand, since parent_sampled={self.parent_sampled} "
  432. f"and sample_rate={sample_rate}."
  433. )
  434. return
  435. self.dynamic_sampling_context["sample_rand"] = f"{sample_rand:.6f}" # noqa: E231
  436. def _sample_rand(self):
  437. # type: () -> Optional[str]
  438. """Convenience method to get the sample_rand value from the dynamic_sampling_context."""
  439. if self.dynamic_sampling_context is None:
  440. return None
  441. return self.dynamic_sampling_context.get("sample_rand")
  442. class Baggage:
  443. """
  444. The W3C Baggage header information (see https://www.w3.org/TR/baggage/).
  445. Before mutating a `Baggage` object, calling code must check that `mutable` is `True`.
  446. Mutating a `Baggage` object that has `mutable` set to `False` is not allowed, but
  447. it is the caller's responsibility to enforce this restriction.
  448. """
  449. __slots__ = ("sentry_items", "third_party_items", "mutable")
  450. SENTRY_PREFIX = "sentry-"
  451. SENTRY_PREFIX_REGEX = re.compile("^sentry-")
  452. def __init__(
  453. self,
  454. sentry_items, # type: Dict[str, str]
  455. third_party_items="", # type: str
  456. mutable=True, # type: bool
  457. ):
  458. self.sentry_items = sentry_items
  459. self.third_party_items = third_party_items
  460. self.mutable = mutable
  461. @classmethod
  462. def from_incoming_header(
  463. cls,
  464. header, # type: Optional[str]
  465. *,
  466. _sample_rand=None, # type: Optional[str]
  467. ):
  468. # type: (...) -> Baggage
  469. """
  470. freeze if incoming header already has sentry baggage
  471. """
  472. sentry_items = {}
  473. third_party_items = ""
  474. mutable = True
  475. if header:
  476. for item in header.split(","):
  477. if "=" not in item:
  478. continue
  479. with capture_internal_exceptions():
  480. item = item.strip()
  481. key, val = item.split("=")
  482. if Baggage.SENTRY_PREFIX_REGEX.match(key):
  483. baggage_key = unquote(key.split("-")[1])
  484. sentry_items[baggage_key] = unquote(val)
  485. mutable = False
  486. else:
  487. third_party_items += ("," if third_party_items else "") + item
  488. if _sample_rand is not None:
  489. sentry_items["sample_rand"] = str(_sample_rand)
  490. mutable = False
  491. return Baggage(sentry_items, third_party_items, mutable)
  492. @classmethod
  493. def from_options(cls, scope):
  494. # type: (sentry_sdk.scope.Scope) -> Optional[Baggage]
  495. sentry_items = {} # type: Dict[str, str]
  496. third_party_items = ""
  497. mutable = False
  498. client = sentry_sdk.get_client()
  499. if not client.is_active() or scope._propagation_context is None:
  500. return Baggage(sentry_items)
  501. options = client.options
  502. propagation_context = scope._propagation_context
  503. if propagation_context is not None:
  504. sentry_items["trace_id"] = propagation_context.trace_id
  505. if options.get("environment"):
  506. sentry_items["environment"] = options["environment"]
  507. if options.get("release"):
  508. sentry_items["release"] = options["release"]
  509. if options.get("dsn"):
  510. sentry_items["public_key"] = Dsn(options["dsn"]).public_key
  511. if options.get("traces_sample_rate"):
  512. sentry_items["sample_rate"] = str(options["traces_sample_rate"])
  513. return Baggage(sentry_items, third_party_items, mutable)
  514. @classmethod
  515. def populate_from_transaction(cls, transaction):
  516. # type: (sentry_sdk.tracing.Transaction) -> Baggage
  517. """
  518. Populate fresh baggage entry with sentry_items and make it immutable
  519. if this is the head SDK which originates traces.
  520. """
  521. client = sentry_sdk.get_client()
  522. sentry_items = {} # type: Dict[str, str]
  523. if not client.is_active():
  524. return Baggage(sentry_items)
  525. options = client.options or {}
  526. sentry_items["trace_id"] = transaction.trace_id
  527. sentry_items["sample_rand"] = f"{transaction._sample_rand:.6f}" # noqa: E231
  528. if options.get("environment"):
  529. sentry_items["environment"] = options["environment"]
  530. if options.get("release"):
  531. sentry_items["release"] = options["release"]
  532. if options.get("dsn"):
  533. sentry_items["public_key"] = Dsn(options["dsn"]).public_key
  534. if (
  535. transaction.name
  536. and transaction.source not in LOW_QUALITY_TRANSACTION_SOURCES
  537. ):
  538. sentry_items["transaction"] = transaction.name
  539. if transaction.sample_rate is not None:
  540. sentry_items["sample_rate"] = str(transaction.sample_rate)
  541. if transaction.sampled is not None:
  542. sentry_items["sampled"] = "true" if transaction.sampled else "false"
  543. # there's an existing baggage but it was mutable,
  544. # which is why we are creating this new baggage.
  545. # However, if by chance the user put some sentry items in there, give them precedence.
  546. if transaction._baggage and transaction._baggage.sentry_items:
  547. sentry_items.update(transaction._baggage.sentry_items)
  548. return Baggage(sentry_items, mutable=False)
  549. def freeze(self):
  550. # type: () -> None
  551. self.mutable = False
  552. def dynamic_sampling_context(self):
  553. # type: () -> Dict[str, str]
  554. header = {}
  555. for key, item in self.sentry_items.items():
  556. header[key] = item
  557. return header
  558. def serialize(self, include_third_party=False):
  559. # type: (bool) -> str
  560. items = []
  561. for key, val in self.sentry_items.items():
  562. with capture_internal_exceptions():
  563. item = Baggage.SENTRY_PREFIX + quote(key) + "=" + quote(str(val))
  564. items.append(item)
  565. if include_third_party:
  566. items.append(self.third_party_items)
  567. return ",".join(items)
  568. @staticmethod
  569. def strip_sentry_baggage(header):
  570. # type: (str) -> str
  571. """Remove Sentry baggage from the given header.
  572. Given a Baggage header, return a new Baggage header with all Sentry baggage items removed.
  573. """
  574. return ",".join(
  575. (
  576. item
  577. for item in header.split(",")
  578. if not Baggage.SENTRY_PREFIX_REGEX.match(item.strip())
  579. )
  580. )
  581. def _sample_rand(self):
  582. # type: () -> Optional[float]
  583. """Convenience method to get the sample_rand value from the sentry_items.
  584. We validate the value and parse it as a float before returning it. The value is considered
  585. valid if it is a float in the range [0, 1).
  586. """
  587. sample_rand = try_convert(float, self.sentry_items.get("sample_rand"))
  588. if sample_rand is not None and 0.0 <= sample_rand < 1.0:
  589. return sample_rand
  590. return None
  591. def __repr__(self):
  592. # type: () -> str
  593. return f'<Baggage "{self.serialize(include_third_party=True)}", mutable={self.mutable}>'
  594. def should_propagate_trace(client, url):
  595. # type: (sentry_sdk.client.BaseClient, str) -> bool
  596. """
  597. Returns True if url matches trace_propagation_targets configured in the given client. Otherwise, returns False.
  598. """
  599. trace_propagation_targets = client.options["trace_propagation_targets"]
  600. if is_sentry_url(client, url):
  601. return False
  602. return match_regex_list(url, trace_propagation_targets, substring_matching=True)
  603. def normalize_incoming_data(incoming_data):
  604. # type: (Dict[str, Any]) -> Dict[str, Any]
  605. """
  606. Normalizes incoming data so the keys are all lowercase with dashes instead of underscores and stripped from known prefixes.
  607. """
  608. data = {}
  609. for key, value in incoming_data.items():
  610. if key.startswith("HTTP_"):
  611. key = key[5:]
  612. key = key.replace("_", "-").lower()
  613. data[key] = value
  614. return data
  615. def create_span_decorator(
  616. op=None, name=None, attributes=None, template=SPANTEMPLATE.DEFAULT
  617. ):
  618. # type: (Optional[Union[str, OP]], Optional[str], Optional[dict[str, Any]], SPANTEMPLATE) -> Any
  619. """
  620. Create a span decorator that can wrap both sync and async functions.
  621. :param op: The operation type for the span.
  622. :type op: str or :py:class:`sentry_sdk.consts.OP` or None
  623. :param name: The name of the span.
  624. :type name: str or None
  625. :param attributes: Additional attributes to set on the span.
  626. :type attributes: dict or None
  627. :param template: The type of span to create. This determines what kind of
  628. span instrumentation and data collection will be applied. Use predefined
  629. constants from :py:class:`sentry_sdk.consts.SPANTEMPLATE`.
  630. The default is `SPANTEMPLATE.DEFAULT` which is the right choice for most
  631. use cases.
  632. :type template: :py:class:`sentry_sdk.consts.SPANTEMPLATE`
  633. """
  634. from sentry_sdk.scope import should_send_default_pii
  635. def span_decorator(f):
  636. # type: (Any) -> Any
  637. """
  638. Decorator to create a span for the given function.
  639. """
  640. @functools.wraps(f)
  641. async def async_wrapper(*args, **kwargs):
  642. # type: (*Any, **Any) -> Any
  643. current_span = get_current_span()
  644. if current_span is None:
  645. logger.debug(
  646. "Cannot create a child span for %s. "
  647. "Please start a Sentry transaction before calling this function.",
  648. qualname_from_function(f),
  649. )
  650. return await f(*args, **kwargs)
  651. span_op = op or _get_span_op(template)
  652. function_name = name or qualname_from_function(f) or ""
  653. span_name = _get_span_name(template, function_name, kwargs)
  654. send_pii = should_send_default_pii()
  655. with current_span.start_child(
  656. op=span_op,
  657. name=span_name,
  658. ) as span:
  659. span.update_data(attributes or {})
  660. _set_input_attributes(
  661. span, template, send_pii, function_name, f, args, kwargs
  662. )
  663. result = await f(*args, **kwargs)
  664. _set_output_attributes(span, template, send_pii, result)
  665. return result
  666. try:
  667. async_wrapper.__signature__ = inspect.signature(f) # type: ignore[attr-defined]
  668. except Exception:
  669. pass
  670. @functools.wraps(f)
  671. def sync_wrapper(*args, **kwargs):
  672. # type: (*Any, **Any) -> Any
  673. current_span = get_current_span()
  674. if current_span is None:
  675. logger.debug(
  676. "Cannot create a child span for %s. "
  677. "Please start a Sentry transaction before calling this function.",
  678. qualname_from_function(f),
  679. )
  680. return f(*args, **kwargs)
  681. span_op = op or _get_span_op(template)
  682. function_name = name or qualname_from_function(f) or ""
  683. span_name = _get_span_name(template, function_name, kwargs)
  684. send_pii = should_send_default_pii()
  685. with current_span.start_child(
  686. op=span_op,
  687. name=span_name,
  688. ) as span:
  689. span.update_data(attributes or {})
  690. _set_input_attributes(
  691. span, template, send_pii, function_name, f, args, kwargs
  692. )
  693. result = f(*args, **kwargs)
  694. _set_output_attributes(span, template, send_pii, result)
  695. return result
  696. try:
  697. sync_wrapper.__signature__ = inspect.signature(f) # type: ignore[attr-defined]
  698. except Exception:
  699. pass
  700. if inspect.iscoroutinefunction(f):
  701. return async_wrapper
  702. else:
  703. return sync_wrapper
  704. return span_decorator
  705. def get_current_span(scope=None):
  706. # type: (Optional[sentry_sdk.Scope]) -> Optional[Span]
  707. """
  708. Returns the currently active span if there is one running, otherwise `None`
  709. """
  710. scope = scope or sentry_sdk.get_current_scope()
  711. current_span = scope.span
  712. return current_span
  713. def set_span_errored(span=None):
  714. # type: (Optional[Span]) -> None
  715. """
  716. Set the status of the current or given span to ERROR.
  717. Also sets the status of the transaction (root span) to ERROR.
  718. """
  719. span = span or get_current_span()
  720. if span is not None:
  721. span.set_status(SPANSTATUS.ERROR)
  722. if span.containing_transaction is not None:
  723. span.containing_transaction.set_status(SPANSTATUS.ERROR)
  724. def _generate_sample_rand(
  725. trace_id, # type: Optional[str]
  726. *,
  727. interval=(0.0, 1.0), # type: tuple[float, float]
  728. ):
  729. # type: (...) -> float
  730. """Generate a sample_rand value from a trace ID.
  731. The generated value will be pseudorandomly chosen from the provided
  732. interval. Specifically, given (lower, upper) = interval, the generated
  733. value will be in the range [lower, upper). The value has 6-digit precision,
  734. so when printing with .6f, the value will never be rounded up.
  735. The pseudorandom number generator is seeded with the trace ID.
  736. """
  737. lower, upper = interval
  738. if not lower < upper: # using `if lower >= upper` would handle NaNs incorrectly
  739. raise ValueError("Invalid interval: lower must be less than upper")
  740. rng = Random(trace_id)
  741. lower_scaled = int(lower * 1_000_000)
  742. upper_scaled = int(upper * 1_000_000)
  743. try:
  744. sample_rand_scaled = rng.randrange(lower_scaled, upper_scaled)
  745. except ValueError:
  746. # In some corner cases it might happen that the range is too small
  747. # In that case, just take the lower bound
  748. sample_rand_scaled = lower_scaled
  749. return sample_rand_scaled / 1_000_000
  750. def _sample_rand_range(parent_sampled, sample_rate):
  751. # type: (Optional[bool], Optional[float]) -> tuple[float, float]
  752. """
  753. Compute the lower (inclusive) and upper (exclusive) bounds of the range of values
  754. that a generated sample_rand value must fall into, given the parent_sampled and
  755. sample_rate values.
  756. """
  757. if parent_sampled is None or sample_rate is None:
  758. return 0.0, 1.0
  759. elif parent_sampled is True:
  760. return 0.0, sample_rate
  761. else: # parent_sampled is False
  762. return sample_rate, 1.0
  763. def _get_value(source, key):
  764. # type: (Any, str) -> Optional[Any]
  765. """
  766. Gets a value from a source object. The source can be a dict or an object.
  767. It is checked for dictionary keys and object attributes.
  768. """
  769. value = None
  770. if isinstance(source, dict):
  771. value = source.get(key)
  772. else:
  773. if hasattr(source, key):
  774. try:
  775. value = getattr(source, key)
  776. except Exception:
  777. value = None
  778. return value
  779. def _get_span_name(template, name, kwargs=None):
  780. # type: (Union[str, SPANTEMPLATE], str, Optional[dict[str, Any]]) -> str
  781. """
  782. Get the name of the span based on the template and the name.
  783. """
  784. span_name = name
  785. if template == SPANTEMPLATE.AI_CHAT:
  786. model = None
  787. if kwargs:
  788. for key in ("model", "model_name"):
  789. if kwargs.get(key) and isinstance(kwargs[key], str):
  790. model = kwargs[key]
  791. break
  792. span_name = f"chat {model}" if model else "chat"
  793. elif template == SPANTEMPLATE.AI_AGENT:
  794. span_name = f"invoke_agent {name}"
  795. elif template == SPANTEMPLATE.AI_TOOL:
  796. span_name = f"execute_tool {name}"
  797. return span_name
  798. def _get_span_op(template):
  799. # type: (Union[str, SPANTEMPLATE]) -> str
  800. """
  801. Get the operation of the span based on the template.
  802. """
  803. mapping = {
  804. SPANTEMPLATE.AI_CHAT: OP.GEN_AI_CHAT,
  805. SPANTEMPLATE.AI_AGENT: OP.GEN_AI_INVOKE_AGENT,
  806. SPANTEMPLATE.AI_TOOL: OP.GEN_AI_EXECUTE_TOOL,
  807. } # type: dict[Union[str, SPANTEMPLATE], Union[str, OP]]
  808. op = mapping.get(template, OP.FUNCTION)
  809. return str(op)
  810. def _get_input_attributes(template, send_pii, args, kwargs):
  811. # type: (Union[str, SPANTEMPLATE], bool, tuple[Any, ...], dict[str, Any]) -> dict[str, Any]
  812. """
  813. Get input attributes for the given span template.
  814. """
  815. attributes = {} # type: dict[str, Any]
  816. if template in [SPANTEMPLATE.AI_AGENT, SPANTEMPLATE.AI_TOOL, SPANTEMPLATE.AI_CHAT]:
  817. mapping = {
  818. "model": (SPANDATA.GEN_AI_REQUEST_MODEL, str),
  819. "model_name": (SPANDATA.GEN_AI_REQUEST_MODEL, str),
  820. "agent": (SPANDATA.GEN_AI_AGENT_NAME, str),
  821. "agent_name": (SPANDATA.GEN_AI_AGENT_NAME, str),
  822. "max_tokens": (SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, int),
  823. "frequency_penalty": (SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, float),
  824. "presence_penalty": (SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, float),
  825. "temperature": (SPANDATA.GEN_AI_REQUEST_TEMPERATURE, float),
  826. "top_p": (SPANDATA.GEN_AI_REQUEST_TOP_P, float),
  827. "top_k": (SPANDATA.GEN_AI_REQUEST_TOP_K, int),
  828. }
  829. def _set_from_key(key, value):
  830. # type: (str, Any) -> None
  831. if key in mapping:
  832. (attribute, data_type) = mapping[key]
  833. if value is not None and isinstance(value, data_type):
  834. attributes[attribute] = value
  835. for key, value in list(kwargs.items()):
  836. if key == "prompt" and isinstance(value, str):
  837. attributes.setdefault(SPANDATA.GEN_AI_REQUEST_MESSAGES, []).append(
  838. {"role": "user", "content": value}
  839. )
  840. continue
  841. if key == "system_prompt" and isinstance(value, str):
  842. attributes.setdefault(SPANDATA.GEN_AI_REQUEST_MESSAGES, []).append(
  843. {"role": "system", "content": value}
  844. )
  845. continue
  846. _set_from_key(key, value)
  847. if template == SPANTEMPLATE.AI_TOOL and send_pii:
  848. attributes[SPANDATA.GEN_AI_TOOL_INPUT] = safe_repr(
  849. {"args": args, "kwargs": kwargs}
  850. )
  851. # Coerce to string
  852. if SPANDATA.GEN_AI_REQUEST_MESSAGES in attributes:
  853. attributes[SPANDATA.GEN_AI_REQUEST_MESSAGES] = safe_repr(
  854. attributes[SPANDATA.GEN_AI_REQUEST_MESSAGES]
  855. )
  856. return attributes
  857. def _get_usage_attributes(usage):
  858. # type: (Any) -> dict[str, Any]
  859. """
  860. Get usage attributes.
  861. """
  862. attributes = {}
  863. def _set_from_keys(attribute, keys):
  864. # type: (str, tuple[str, ...]) -> None
  865. for key in keys:
  866. value = _get_value(usage, key)
  867. if value is not None and isinstance(value, int):
  868. attributes[attribute] = value
  869. _set_from_keys(
  870. SPANDATA.GEN_AI_USAGE_INPUT_TOKENS,
  871. ("prompt_tokens", "input_tokens"),
  872. )
  873. _set_from_keys(
  874. SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS,
  875. ("completion_tokens", "output_tokens"),
  876. )
  877. _set_from_keys(
  878. SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS,
  879. ("total_tokens",),
  880. )
  881. return attributes
  882. def _get_output_attributes(template, send_pii, result):
  883. # type: (Union[str, SPANTEMPLATE], bool, Any) -> dict[str, Any]
  884. """
  885. Get output attributes for the given span template.
  886. """
  887. attributes = {} # type: dict[str, Any]
  888. if template in [SPANTEMPLATE.AI_AGENT, SPANTEMPLATE.AI_TOOL, SPANTEMPLATE.AI_CHAT]:
  889. with capture_internal_exceptions():
  890. # Usage from result, result.usage, and result.metadata.usage
  891. usage_candidates = [result]
  892. usage = _get_value(result, "usage")
  893. usage_candidates.append(usage)
  894. meta = _get_value(result, "metadata")
  895. usage = _get_value(meta, "usage")
  896. usage_candidates.append(usage)
  897. for usage_candidate in usage_candidates:
  898. if usage_candidate is not None:
  899. attributes.update(_get_usage_attributes(usage_candidate))
  900. # Response model
  901. model_name = _get_value(result, "model")
  902. if model_name is not None and isinstance(model_name, str):
  903. attributes[SPANDATA.GEN_AI_RESPONSE_MODEL] = model_name
  904. model_name = _get_value(result, "model_name")
  905. if model_name is not None and isinstance(model_name, str):
  906. attributes[SPANDATA.GEN_AI_RESPONSE_MODEL] = model_name
  907. # Tool output
  908. if template == SPANTEMPLATE.AI_TOOL and send_pii:
  909. attributes[SPANDATA.GEN_AI_TOOL_OUTPUT] = safe_repr(result)
  910. return attributes
  911. def _set_input_attributes(span, template, send_pii, name, f, args, kwargs):
  912. # type: (Span, Union[str, SPANTEMPLATE], bool, str, Any, tuple[Any, ...], dict[str, Any]) -> None
  913. """
  914. Set span input attributes based on the given span template.
  915. :param span: The span to set attributes on.
  916. :param template: The template to use to set attributes on the span.
  917. :param send_pii: Whether to send PII data.
  918. :param f: The wrapped function.
  919. :param args: The arguments to the wrapped function.
  920. :param kwargs: The keyword arguments to the wrapped function.
  921. """
  922. attributes = {} # type: dict[str, Any]
  923. if template == SPANTEMPLATE.AI_AGENT:
  924. attributes = {
  925. SPANDATA.GEN_AI_OPERATION_NAME: "invoke_agent",
  926. SPANDATA.GEN_AI_AGENT_NAME: name,
  927. }
  928. elif template == SPANTEMPLATE.AI_CHAT:
  929. attributes = {
  930. SPANDATA.GEN_AI_OPERATION_NAME: "chat",
  931. }
  932. elif template == SPANTEMPLATE.AI_TOOL:
  933. attributes = {
  934. SPANDATA.GEN_AI_OPERATION_NAME: "execute_tool",
  935. SPANDATA.GEN_AI_TOOL_NAME: name,
  936. }
  937. docstring = f.__doc__
  938. if docstring is not None:
  939. attributes[SPANDATA.GEN_AI_TOOL_DESCRIPTION] = docstring
  940. attributes.update(_get_input_attributes(template, send_pii, args, kwargs))
  941. span.update_data(attributes or {})
  942. def _set_output_attributes(span, template, send_pii, result):
  943. # type: (Span, Union[str, SPANTEMPLATE], bool, Any) -> None
  944. """
  945. Set span output attributes based on the given span template.
  946. :param span: The span to set attributes on.
  947. :param template: The template to use to set attributes on the span.
  948. :param send_pii: Whether to send PII data.
  949. :param result: The result of the wrapped function.
  950. """
  951. span.update_data(_get_output_attributes(template, send_pii, result) or {})
  952. # Circular imports
  953. from sentry_sdk.tracing import (
  954. BAGGAGE_HEADER_NAME,
  955. LOW_QUALITY_TRANSACTION_SOURCES,
  956. SENTRY_TRACE_HEADER_NAME,
  957. )
  958. if TYPE_CHECKING:
  959. from sentry_sdk.tracing import Span