[dbsrgits/DBIx-Class.git] / t / lib / DBICTest / Util / LeakTracer.pm

package DBICTest::Util::LeakTracer;

use warnings;
use strict;

use Carp;
use Scalar::Util qw(isweak weaken blessed reftype);
use DBIx::Class::_Util 'refcount';
use DBIx::Class::Optional::Dependencies;
use Data::Dumper::Concise;
use DBICTest::Util 'stacktrace';
use constant {
  CV_TRACING => DBIx::Class::Optional::Dependencies->req_ok_for ('test_leaks_heavy'),
  SKIP_SCALAR_REFS => ( $] > 5.017 ) ? 1 : 0,
};

use base 'Exporter';
our @EXPORT_OK = qw(populate_weakregistry assert_empty_weakregistry hrefaddr visit_refs);

my $refs_traced = 0;
my $leaks_found = 0;
my %reg_of_regs;

sub hrefaddr { sprintf '0x%x', &Scalar::Util::refaddr }

# so we don't trigger stringification
sub _describe_ref {
  sprintf '%s%s(%s)',
    (defined blessed $_[0]) ? blessed($_[0]) . '=' : '',
    reftype $_[0],
    hrefaddr $_[0],
  ;
}

sub populate_weakregistry {
  my ($weak_registry, $target, $note) = @_;

  croak 'Expecting a registry hashref' unless ref $weak_registry eq 'HASH';
  croak 'Target is not a reference' unless length ref $target;

  my $refaddr = hrefaddr $target;

  # a registry could be fed to itself or another registry via recursive sweeps
  return $target if $reg_of_regs{$refaddr};

  weaken( $reg_of_regs{ hrefaddr($weak_registry) } = $weak_registry )
    unless( $reg_of_regs{ hrefaddr($weak_registry) } );

  # an explicit "garbage collection" pass every time we store a ref
  # if we do not do this the registry will keep growing appearing
  # as if the traced program is continuously slowly leaking memory
  for my $reg (values %reg_of_regs) {
    (defined $reg->{$_}{weakref}) or delete $reg->{$_}
      for keys %$reg;
  }

  # FIXME/INVESTIGATE - something fishy is going on with refs to plain
  # strings, perhaps something to do with the CoW work etc...
  return $target if SKIP_SCALAR_REFS and reftype($target) eq 'SCALAR';

  if (! defined $weak_registry->{$refaddr}{weakref}) {
    $weak_registry->{$refaddr} = {
      stacktrace => stacktrace(1),
      weakref => $target,
    };
    weaken( $weak_registry->{$refaddr}{weakref} );
    $refs_traced++;
  }

  my $desc = _describe_ref($target);
  $weak_registry->{$refaddr}{slot_names}{$desc} = 1;
  if ($note) {
    $note =~ s/\s*\Q$desc\E\s*//g;
    $weak_registry->{$refaddr}{slot_names}{$note} = 1;
  }

  $target;
}

# Regenerate the slots names on a thread spawn
sub CLONE {
  my @individual_regs = grep { scalar keys %{$_||{}} } values %reg_of_regs;
  %reg_of_regs = ();

  for my $reg (@individual_regs) {
    my @live_slots = grep { defined $_->{weakref} } values %$reg
      or next;

    $reg = {};  # get a fresh hashref in the new thread ctx
    weaken( $reg_of_regs{hrefaddr($reg)} = $reg );

    for my $slot_info (@live_slots) {
      my $new_addr = hrefaddr $slot_info->{weakref};

      # replace all slot names
      $slot_info->{slot_names} = { map {
        my $name = $_;
        $name =~ s/\(0x[0-9A-F]+\)/sprintf ('(%s)', $new_addr)/ieg;
        ($name => 1);
      } keys %{$slot_info->{slot_names}} };

      $reg->{$new_addr} = $slot_info;
    }
  }
}

sub visit_refs {
  my $args = { (ref $_[0]) ? %{$_[0]} : @_ };

  $args->{seen_refs} ||= {};

  my $visited_cnt = '0E0';
  for my $i (0 .. $#{$args->{refs}} ) {
    next if isweak($args->{refs}[$i]);

    my $r = $args->{refs}[$i];

    next unless length ref $r;

    next if $args->{seen_refs}{my $dec_addr = Scalar::Util::refaddr($r)}++;

    $visited_cnt++;
    $args->{action}->($r) or next;

    my $type = reftype $r;
    if ($type eq 'HASH') {
      $visited_cnt += visit_refs({ %$args, refs => [ map {
        ( !isweak($r->{$_}) ) ? $r->{$_} : ()
      } keys %$r ] });
    }
    elsif ($type eq 'ARRAY') {
      $visited_cnt += visit_refs({ %$args, refs => [ map {
        ( !isweak($r->[$_]) ) ? $r->[$_] : ()
      } 0..$#$r ] });
    }
    elsif ($type eq 'REF' and !isweak($$r)) {
      $visited_cnt += visit_refs({ %$args, refs => [ $$r ] });
    }
    elsif (CV_TRACING and $type eq 'CODE') {
      $visited_cnt += visit_refs({ %$args, refs => [ map {
        ( !isweak($_) ) ? $_ : ()
      } scalar PadWalker::closed_over($r) ] }); # scalar due to RT#92269
    }
  }
  $visited_cnt;
}

sub assert_empty_weakregistry {
  my ($weak_registry, $quiet) = @_;

  # in case we hooked bless any extra object creation will wreak
  # havoc during the assert phase
  local *CORE::GLOBAL::bless;
  *CORE::GLOBAL::bless = sub { CORE::bless( $_[0], (@_ > 1) ? $_[1] : caller() ) };

  croak 'Expecting a registry hashref' unless ref $weak_registry eq 'HASH';

  return unless keys %$weak_registry;

  my $tb = eval { Test::Builder->new }
    or croak "Calling assert_empty_weakregistry in $0 without a loaded Test::Builder makes no sense";

  for my $addr (keys %$weak_registry) {
    $weak_registry->{$addr}{display_name} = join ' | ', (
      sort
        { length $a <=> length $b or $a cmp $b }
        keys %{$weak_registry->{$addr}{slot_names}}
    );

    $tb->BAILOUT("!!!! WEAK REGISTRY SLOT $weak_registry->{$addr}{display_name} IS NOT A WEAKREF !!!!")
      if defined $weak_registry->{$addr}{weakref} and ! isweak( $weak_registry->{$addr}{weakref} );
  }

  # compile a list of refs stored as globals (possibly even catching
  # class data in the form of method closures), so we can skip them
  # further on
  my ($seen_refs, $classdata_refs) = ({}, undef);

  # the walk is very expensive - if we are $quiet (running in an END block)
  # we do not really need to be too thorough
  unless ($quiet) {
    my ($symwalker, $symcounts);
    $symwalker = sub {
      no strict 'refs';
      my $pkg = shift || '::';

      # any non-weak globals are "clasdata" in all possible sense
      #
      # the unless regex at the end skips some dangerous namespaces outright
      # (but does not prevent descent)
      $symcounts->{$pkg} += visit_refs (
        seen_refs => $seen_refs,
        action => sub { ++$classdata_refs->{hrefaddr $_[0]} },
        refs => [ map { my $sym = $_;
          # *{"$pkg$sym"}{CODE} won't simply work - MRO-cached CVs are invisible there
          ( CV_TRACING ? Class::MethodCache::get_cv("${pkg}$sym") : () ),

          ( defined *{"$pkg$sym"}{SCALAR} and length ref ${"$pkg$sym"} and ! isweak( ${"$pkg$sym"} ) )
            ? ${"$pkg$sym"} : ()
          ,
          ( map {
            ( defined *{"$pkg$sym"}{$_} and ! isweak(defined *{"$pkg$sym"}{$_}) )
              ? *{"$pkg$sym"}{$_}
              : ()
          } qw(HASH ARRAY IO GLOB) ),
        } keys %$pkg ],
      ) unless $pkg =~ /^ :: (?:
        DB | next | B | .+? ::::ISA (?: ::CACHE ) | Class::C3
      ) :: $/x;

      $symwalker->("${pkg}$_") for grep { $_ =~ /(?<!^main)::$/ } keys %$pkg;
    };

    $symwalker->();

#    use Devel::Dwarn;
#    Ddie [ map
#      { { $_ => $symcounts->{$_} } }
#      sort
#        {$symcounts->{$a} <=> $symcounts->{$b} }
#        keys %$symcounts
#    ];
  }

  delete $weak_registry->{$_} for keys %$classdata_refs;

  for my $addr (sort { $weak_registry->{$a}{display_name} cmp $weak_registry->{$b}{display_name} } keys %$weak_registry) {

    next if ! defined $weak_registry->{$addr}{weakref};

    $leaks_found++ unless $tb->in_todo;
    $tb->ok (0, "Leaked $weak_registry->{$addr}{display_name}");

    my $diag = do {
      local $Data::Dumper::Maxdepth = 1;
      sprintf "\n%s (refcnt %d) => %s\n",
        $weak_registry->{$addr}{display_name},
        refcount($weak_registry->{$addr}{weakref}),
        (
          ref($weak_registry->{$addr}{weakref}) eq 'CODE'
            and
          B::svref_2object($weak_registry->{$addr}{weakref})->XSUB
        ) ? '__XSUB__' : Dumper( $weak_registry->{$addr}{weakref} )
      ;
    };

    # FIXME - need to add a circular reference seeker based on the visitor
    # (will need a bunch of modifications, punting with just a stub for now)

    $diag .= Devel::FindRef::track ($weak_registry->{$addr}{weakref}, 50) . "\n"
      if ( $ENV{TEST_VERBOSE} && eval { require Devel::FindRef });

    $diag =~ s/^/    /mg;

    if (my $stack = $weak_registry->{$addr}{stacktrace}) {
      $diag .= "    Reference first seen$stack";
    }

    $tb->diag($diag);
  }

  if (! $quiet and !$leaks_found and ! $tb->in_todo) {
    $tb->ok(1, sprintf "No leaks found at %s line %d", (caller())[1,2] );
  }
}

END {
  if ($INC{'Test/Builder.pm'}) {
    my $tb = Test::Builder->new;

    # we check for test passage - a leak may be a part of a TODO
    if ($leaks_found and !$tb->is_passing) {

      $tb->diag(sprintf
        "\n\n%s\n%s\n\nInstall Devel::FindRef and re-run the test with set "
      . '$ENV{TEST_VERBOSE} (prove -v) to see a more detailed leak-report'
      . "\n\n%s\n%s\n\n", ('#' x 16) x 4
      ) if ( !$ENV{TEST_VERBOSE} or !$INC{'Devel/FindRef.pm'} );

    }
    else {
      $tb->note("Auto checked $refs_traced references for leaks - none detected");
    }
  }
}

1;
Commit	Line	Data
218b7c12	1	package DBICTest::Util::LeakTracer;
	2
	3	use warnings;
	4	use strict;
	5
	6	use Carp;
96577657	7	use Scalar::Util qw(isweak weaken blessed reftype);
dac7972a	8	use DBIx::Class::_Util 'refcount';
556c4fe6	9	use DBIx::Class::Optional::Dependencies;
96577657	10	use Data::Dumper::Concise;
218b7c12	11	use DBICTest::Util 'stacktrace';
556c4fe6	12	use constant {
556c4fe6	13	CV_TRACING => DBIx::Class::Optional::Dependencies->req_ok_for ('test_leaks_heavy'),
10635a06	14	SKIP_SCALAR_REFS => ( $] > 5.017 ) ? 1 : 0,
556c4fe6	15	};
218b7c12	16
218b7c12	17	use base 'Exporter';
21aa86aa	18	our @EXPORT_OK = qw(populate_weakregistry assert_empty_weakregistry hrefaddr visit_refs);
218b7c12	19
218b7c12	20	my $refs_traced = 0;
96577657	21	my $leaks_found = 0;
218b7c12	22	my %reg_of_regs;
218b7c12	23
96577657	24	sub hrefaddr { sprintf '0x%x', &Scalar::Util::refaddr }
	25
	26	# so we don't trigger stringification
	27	sub _describe_ref {
	28	sprintf '%s%s(%s)',
	29	(defined blessed $_[0]) ? blessed($_[0]) . '=' : '',
	30	reftype $_[0],
	31	hrefaddr $_[0],
	32	;
	33	}
	34
218b7c12	35	sub populate_weakregistry {
96577657	36	my ($weak_registry, $target, $note) = @_;
218b7c12	37
	38	croak 'Expecting a registry hashref' unless ref $weak_registry eq 'HASH';
	39	croak 'Target is not a reference' unless length ref $target;
	40
96577657	41	my $refaddr = hrefaddr $target;
8fa57d17	42
96577657	43	# a registry could be fed to itself or another registry via recursive sweeps
96577657	44	return $target if $reg_of_regs{$refaddr};
218b7c12	45
85ad63df	46	weaken( $reg_of_regs{ hrefaddr($weak_registry) } = $weak_registry )
	47	unless( $reg_of_regs{ hrefaddr($weak_registry) } );
	48
	49	# an explicit "garbage collection" pass every time we store a ref
	50	# if we do not do this the registry will keep growing appearing
	51	# as if the traced program is continuously slowly leaking memory
	52	for my $reg (values %reg_of_regs) {
	53	(defined $reg->{$_}{weakref}) or delete $reg->{$_}
	54	for keys %$reg;
	55	}
	56
10635a06	57	# FIXME/INVESTIGATE - something fishy is going on with refs to plain
	58	# strings, perhaps something to do with the CoW work etc...
	59	return $target if SKIP_SCALAR_REFS and reftype($target) eq 'SCALAR';
	60
96577657	61	if (! defined $weak_registry->{$refaddr}{weakref}) {
96577657	62	$weak_registry->{$refaddr} = {
8fa57d17	63	stacktrace => stacktrace(1),
96577657	64	weakref => $target,
8fa57d17	65	};
96577657	66	weaken( $weak_registry->{$refaddr}{weakref} );
8fa57d17	67	$refs_traced++;
218b7c12	68	}
218b7c12	69
96577657	70	my $desc = _describe_ref($target);
	71	$weak_registry->{$refaddr}{slot_names}{$desc} = 1;
	72	if ($note) {
	73	$note =~ s/\s\Q$desc\E\s//g;
	74	$weak_registry->{$refaddr}{slot_names}{$note} = 1;
	75	}
	76
218b7c12	77	$target;
	78	}
	79
96577657	80	# Regenerate the slots names on a thread spawn
218b7c12	81	sub CLONE {
	82	my @individual_regs = grep { scalar keys %{$_\|\|{}} } values %reg_of_regs;
	83	%reg_of_regs = ();
	84
	85	for my $reg (@individual_regs) {
96577657	86	my @live_slots = grep { defined $_->{weakref} } values %$reg
218b7c12	87	or next;
cf8fa286	88
cf8fa286	89	$reg = {}; # get a fresh hashref in the new thread ctx
96577657	90	weaken( $reg_of_regs{hrefaddr($reg)} = $reg );
218b7c12	91
96577657	92	for my $slot_info (@live_slots) {
96577657	93	my $new_addr = hrefaddr $slot_info->{weakref};
8fa57d17	94
96577657	95	# replace all slot names
	96	$slot_info->{slot_names} = { map {
	97	my $name = $_;
	98	$name =~ s/\(0x[0-9A-F]+\)/sprintf ('(%s)', $new_addr)/ieg;
	99	($name => 1);
	100	} keys %{$slot_info->{slot_names}} };
218b7c12	101
96577657	102	$reg->{$new_addr} = $slot_info;
218b7c12	103	}
	104	}
	105	}
	106
556c4fe6	107	sub visit_refs {
	108	my $args = { (ref $_[0]) ? %{$_[0]} : @_ };
	109
	110	$args->{seen_refs} \|\|= {};
	111
	112	my $visited_cnt = '0E0';
	113	for my $i (0 .. $#{$args->{refs}} ) {
	114	next if isweak($args->{refs}[$i]);
	115
	116	my $r = $args->{refs}[$i];
	117
	118	next unless length ref $r;
	119
	120	next if $args->{seen_refs}{my $dec_addr = Scalar::Util::refaddr($r)}++;
	121
	122	$visited_cnt++;
	123	$args->{action}->($r) or next;
	124
	125	my $type = reftype $r;
	126	if ($type eq 'HASH') {
	127	$visited_cnt += visit_refs({ %$args, refs => [ map {
	128	( !isweak($r->{$_}) ) ? $r->{$_} : ()
	129	} keys %$r ] });
	130	}
	131	elsif ($type eq 'ARRAY') {
	132	$visited_cnt += visit_refs({ %$args, refs => [ map {
	133	( !isweak($r->[$_]) ) ? $r->[$_] : ()
	134	} 0..$#$r ] });
	135	}
	136	elsif ($type eq 'REF' and !isweak($$r)) {
	137	$visited_cnt += visit_refs({ %$args, refs => [ $$r ] });
	138	}
	139	elsif (CV_TRACING and $type eq 'CODE') {
	140	$visited_cnt += visit_refs({ %$args, refs => [ map {
	141	( !isweak($_) ) ? $_ : ()
	142	} scalar PadWalker::closed_over($r) ] }); # scalar due to RT#92269
	143	}
	144	}
	145	$visited_cnt;
	146	}
	147
218b7c12	148	sub assert_empty_weakregistry {
	149	my ($weak_registry, $quiet) = @_;
	150
556c4fe6	151	# in case we hooked bless any extra object creation will wreak
	152	# havoc during the assert phase
	153	local *CORE::GLOBAL::bless;
	154	*CORE::GLOBAL::bless = sub { CORE::bless( $_[0], (@_ > 1) ? $_[1] : caller() ) };
	155
218b7c12	156	croak 'Expecting a registry hashref' unless ref $weak_registry eq 'HASH';
	157
	158	return unless keys %$weak_registry;
	159
	160	my $tb = eval { Test::Builder->new }
556c4fe6	161	or croak "Calling assert_empty_weakregistry in $0 without a loaded Test::Builder makes no sense";
218b7c12	162
96577657	163	for my $addr (keys %$weak_registry) {
	164	$weak_registry->{$addr}{display_name} = join ' \| ', (
	165	sort
	166	{ length $a <=> length $b or $a cmp $b }
	167	keys %{$weak_registry->{$addr}{slot_names}}
	168	);
218b7c12	169
96577657	170	$tb->BAILOUT("!!!! WEAK REGISTRY SLOT $weak_registry->{$addr}{display_name} IS NOT A WEAKREF !!!!")
	171	if defined $weak_registry->{$addr}{weakref} and ! isweak( $weak_registry->{$addr}{weakref} );
	172	}
218b7c12	173
556c4fe6	174	# compile a list of refs stored as globals (possibly even catching
	175	# class data in the form of method closures), so we can skip them
	176	# further on
	177	my ($seen_refs, $classdata_refs) = ({}, undef);
	178
	179	# the walk is very expensive - if we are $quiet (running in an END block)
	180	# we do not really need to be too thorough
	181	unless ($quiet) {
	182	my ($symwalker, $symcounts);
	183	$symwalker = sub {
	184	no strict 'refs';
	185	my $pkg = shift \|\| '::';
	186
	187	# any non-weak globals are "clasdata" in all possible sense
	188	#
	189	# the unless regex at the end skips some dangerous namespaces outright
	190	# (but does not prevent descent)
	191	$symcounts->{$pkg} += visit_refs (
	192	seen_refs => $seen_refs,
	193	action => sub { ++$classdata_refs->{hrefaddr $_[0]} },
	194	refs => [ map { my $sym = $_;
	195	# *{"$pkg$sym"}{CODE} won't simply work - MRO-cached CVs are invisible there
	196	( CV_TRACING ? Class::MethodCache::get_cv("${pkg}$sym") : () ),
	197
	198	( defined *{"$pkg$sym"}{SCALAR} and length ref ${"$pkg$sym"} and ! isweak( ${"$pkg$sym"} ) )
	199	? ${"$pkg$sym"} : ()
	200	,
	201	( map {
	202	( defined {"$pkg$sym"}{$_} and ! isweak(defined {"$pkg$sym"}{$_}) )
	203	? *{"$pkg$sym"}{$_}
	204	: ()
	205	} qw(HASH ARRAY IO GLOB) ),
	206	} keys %$pkg ],
	207	) unless $pkg =~ /^ :: (?:
	208	DB \| next \| B \| .+? ::::ISA (?: ::CACHE ) \| Class::C3
	209	) :: $/x;
	210
	211	$symwalker->("${pkg}$_") for grep { $_ =~ /(?<!^main)::$/ } keys %$pkg;
	212	};
8fa57d17	213
	214	$symwalker->();
	215
556c4fe6	216	# use Devel::Dwarn;
	217	# Ddie [ map
	218	# { { $_ => $symcounts->{$_} } }
	219	# sort
	220	# {$symcounts->{$a} <=> $symcounts->{$b} }
	221	# keys %$symcounts
	222	# ];
8fa57d17	223	}
8fa57d17	224
556c4fe6	225	delete $weak_registry->{$_} for keys %$classdata_refs;
556c4fe6	226
96577657	227	for my $addr (sort { $weak_registry->{$a}{display_name} cmp $weak_registry->{$b}{display_name} } keys %$weak_registry) {
96577657	228
1a44a267	229	next if ! defined $weak_registry->{$addr}{weakref};
1a44a267	230
5dc4301c	231	$leaks_found++ unless $tb->in_todo;
1a44a267	232	$tb->ok (0, "Leaked $weak_registry->{$addr}{display_name}");
	233
	234	my $diag = do {
	235	local $Data::Dumper::Maxdepth = 1;
	236	sprintf "\n%s (refcnt %d) => %s\n",
	237	$weak_registry->{$addr}{display_name},
	238	refcount($weak_registry->{$addr}{weakref}),
	239	(
	240	ref($weak_registry->{$addr}{weakref}) eq 'CODE'
	241	and
	242	B::svref_2object($weak_registry->{$addr}{weakref})->XSUB
	243	) ? '__XSUB__' : Dumper( $weak_registry->{$addr}{weakref} )
	244	;
218b7c12	245	};
1a44a267	246
5dc4301c	247	# FIXME - need to add a circular reference seeker based on the visitor
	248	# (will need a bunch of modifications, punting with just a stub for now)
	249
556c4fe6	250	$diag .= Devel::FindRef::track ($weak_registry->{$addr}{weakref}, 50) . "\n"
1a44a267	251	if ( $ENV{TEST_VERBOSE} && eval { require Devel::FindRef });
	252
	253	$diag =~ s/^/ /mg;
	254
	255	if (my $stack = $weak_registry->{$addr}{stacktrace}) {
	256	$diag .= " Reference first seen$stack";
	257	}
	258
	259	$tb->diag($diag);
	260	}
	261
5dc4301c	262	if (! $quiet and !$leaks_found and ! $tb->in_todo) {
1a44a267	263	$tb->ok(1, sprintf "No leaks found at %s line %d", (caller())[1,2] );
218b7c12	264	}
	265	}
	266
	267	END {
	268	if ($INC{'Test/Builder.pm'}) {
	269	my $tb = Test::Builder->new;
	270
	271	# we check for test passage - a leak may be a part of a TODO
	272	if ($leaks_found and !$tb->is_passing) {
	273
	274	$tb->diag(sprintf
	275	"\n\n%s\n%s\n\nInstall Devel::FindRef and re-run the test with set "
	276	. '$ENV{TEST_VERBOSE} (prove -v) to see a more detailed leak-report'
	277	. "\n\n%s\n%s\n\n", ('#' x 16) x 4
	278	) if ( !$ENV{TEST_VERBOSE} or !$INC{'Devel/FindRef.pm'} );
	279
	280	}
	281	else {
	282	$tb->note("Auto checked $refs_traced references for leaks - none detected");
	283	}
	284	}
	285	}
	286
	287	1;