Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24715cc85f | ||
|
|
5cbb9db198 | ||
|
|
2ad0725272 | ||
|
|
79ac8f606a | ||
|
|
01bce22104 | ||
|
|
9ce81bff85 | ||
|
|
746af9947a | ||
|
|
d6bb96e3a1 | ||
|
|
cc39273df2 | ||
|
|
3884cb7b0b | ||
|
|
8667da401c | ||
|
|
e3bc373e2b | ||
|
|
e4287fa88d | ||
|
|
b37d3aa265 | ||
|
|
349581e9d5 | ||
|
|
3b7b26109c | ||
|
|
9a22679c15 | ||
|
|
54eac33780 | ||
|
|
3d73340a29 | ||
|
|
9acda1e1fd | ||
|
|
1525af68c6 | ||
|
|
252ba6ce07 | ||
|
|
39612a0f22 | ||
|
|
9eb13a4658 | ||
|
|
042619c2d7 | ||
|
|
0e92ac04b7 | ||
|
|
b4dd91d975 | ||
|
|
692d26f8cc | ||
|
|
c1d88b56c6 | ||
|
|
d05be13a4e | ||
|
|
085e3f70c9 | ||
|
|
e59f582bbb | ||
|
|
03be06fc3f | ||
|
|
801734b5a5 | ||
|
|
dd74f36d80 | ||
|
|
98bb27047d | ||
|
|
66ea24295b | ||
|
|
cc896fddcc | ||
|
|
94ab356577 | ||
|
|
6f9a710ee8 | ||
|
|
6e7dfd5a75 | ||
|
|
0f991d297f | ||
|
|
bdcf0b0384 | ||
|
|
a4d183ce8e | ||
|
|
c69efdb5f4 | ||
|
|
1a48dd6757 | ||
|
|
9d883347fd | ||
|
|
b9d5d2a699 | ||
|
|
03453fc006 | ||
|
|
e755feaf21 | ||
|
|
18ffdd8c9a | ||
|
|
67511ca09d | ||
|
|
a19d946ebc | ||
|
|
b2ac41c139 | ||
|
|
1cf1f75ee7 | ||
|
|
50eb12a4b3 | ||
|
|
93b5586f6f | ||
|
|
98d59dcdeb | ||
|
|
63142d74e3 | ||
|
|
680d2bf08f | ||
|
|
ecf3c41ca0 | ||
|
|
81285bf85e | ||
|
|
223227bbe5 | ||
|
|
bb8a802b81 | ||
|
|
1aa8abc667 | ||
|
|
ef357765f5 | ||
|
|
09ffc042ed | ||
|
|
8a387e2c25 | ||
|
|
b4bad98892 | ||
|
|
e44325a383 | ||
|
|
158017f839 | ||
|
|
258c38ee4b | ||
|
|
8196f7c7de | ||
|
|
ffdd7efad3 | ||
|
|
0509b471bd | ||
|
|
b0aaccee12 | ||
|
|
27a79341b7 | ||
|
|
fe7dd0a667 | ||
|
|
391f76e291 | ||
|
|
bc97895170 | ||
|
|
9b238f1505 | ||
|
|
66ef904153 | ||
|
|
773f935c43 | ||
|
|
ece91c91a9 | ||
|
|
e6f3908a2e |
@@ -493,6 +493,7 @@ Matt Kraai <kraai@ftbfs.org>
|
||||
Jan Kratochvil <jan.kratochvil@redhat.com>
|
||||
Matthias Kretz <m.kretz@gsi.de>
|
||||
Louis Krupp <louis.krupp@zoho.com>
|
||||
Ondrej Kubanek <kubanek0ondrej@gmail.com>
|
||||
Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org>
|
||||
Venkataramanan Kumar <venkataramanan.kumar@amd.com>
|
||||
Doug Kwan <dougkwan@google.com>
|
||||
@@ -724,6 +725,7 @@ Certificate of Origin Version 1.1. See https://gcc.gnu.org/dco.html for more
|
||||
information.
|
||||
|
||||
|
||||
Ondrej Kubanek <kubanek0ondrej@gmail.com>
|
||||
Matthias Kretz <m.kretz@gsi.de>
|
||||
Tim Lange <mail@tim-lange.me>
|
||||
Jeff Law <jeffreyalaw@gmail.com>
|
||||
|
||||
@@ -1421,6 +1421,7 @@ OBJS = \
|
||||
gimple-loop-interchange.o \
|
||||
gimple-loop-jam.o \
|
||||
gimple-loop-versioning.o \
|
||||
gimple-loop-versioning-histograms.o \
|
||||
gimple-low.o \
|
||||
gimple-predicate-analysis.o \
|
||||
gimple-pretty-print.o \
|
||||
|
||||
+219
-1
@@ -199,6 +199,14 @@ flow_loop_free (class loop *loop)
|
||||
}
|
||||
|
||||
ggc_free (loop->exits);
|
||||
if (loop->counters)
|
||||
{
|
||||
va_heap::release (loop->counters->lin);
|
||||
va_heap::release (loop->counters->exp);
|
||||
if (loop->counters->mod)
|
||||
va_heap::release (loop->counters->mod);
|
||||
ggc_free (loop->counters);
|
||||
}
|
||||
ggc_free (loop);
|
||||
}
|
||||
|
||||
@@ -1770,7 +1778,7 @@ loop_preheader_edge (const class loop *loop)
|
||||
edge_iterator ei;
|
||||
|
||||
gcc_assert (loops_state_satisfies_p (LOOPS_HAVE_PREHEADERS)
|
||||
&& ! loops_state_satisfies_p (LOOPS_MAY_HAVE_MULTIPLE_LATCHES));
|
||||
&& !loops_state_satisfies_p (LOOPS_MAY_HAVE_MULTIPLE_LATCHES));
|
||||
|
||||
FOR_EACH_EDGE (e, ei, loop->header->preds)
|
||||
if (e->src != loop->latch)
|
||||
@@ -2181,3 +2189,213 @@ loops_list::walk_loop_tree (class loop *root, unsigned flags)
|
||||
this->to_visit.quick_push (root->num);
|
||||
}
|
||||
|
||||
unsigned int
|
||||
hist_index (gcov_type_unsigned val, unsigned int lin_size, unsigned int exp_size)
|
||||
{
|
||||
unsigned int tot_size = lin_size + exp_size;
|
||||
if (val < lin_size)
|
||||
{
|
||||
return val;
|
||||
}
|
||||
else
|
||||
{
|
||||
gcov_type_unsigned pow2 = floor_log2 (val);
|
||||
gcov_type_unsigned lin_pow2 = floor_log2 (lin_size - 1);
|
||||
if (lin_size < tot_size && pow2 == lin_pow2)
|
||||
{
|
||||
return lin_size;
|
||||
}
|
||||
if (tot_size > pow2 + ((lin_size - 1) - lin_pow2))
|
||||
{
|
||||
return pow2 + ((lin_size - 1) - lin_pow2);
|
||||
}
|
||||
else
|
||||
{
|
||||
return tot_size - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
histogram_counters_minus_upper_bound (histogram_counters *&hist_c,
|
||||
gcov_type_unsigned difference)
|
||||
{
|
||||
if (difference == 0 || !hist_c)
|
||||
return;
|
||||
if (hist_c->sum == 0 && !hist_c->mod)
|
||||
{
|
||||
va_heap::release (hist_c->lin);
|
||||
va_heap::release (hist_c->exp);
|
||||
ggc_free (hist_c);
|
||||
hist_c = NULL;
|
||||
return;
|
||||
}
|
||||
auto &lin = *(hist_c->lin);
|
||||
auto &exp = *(hist_c->exp);
|
||||
hist_c->adjusted = true;
|
||||
auto &sum = hist_c->sum;
|
||||
unsigned int lin_size = lin.length ();
|
||||
unsigned int exp_size = exp.length ();
|
||||
// If the last linear counter does not contain other iterations
|
||||
unsigned int i = 0;
|
||||
for (; i < difference && i < lin.length (); ++i)
|
||||
{
|
||||
sum -= lin[i];
|
||||
lin[i] = 0;
|
||||
}
|
||||
for (; i < lin.length (); ++i)
|
||||
{
|
||||
lin[i - difference] = lin[i];
|
||||
lin[i] = 0;
|
||||
}
|
||||
lin.truncate (lin.length () > difference ? lin.length () - difference : 0);
|
||||
// next pow2
|
||||
gcov_type_unsigned pow2 = ((gcov_type_unsigned) 1)
|
||||
<< (ceil_log2 (lin_size) + i + 1 - lin_size);
|
||||
i = 0;
|
||||
// we null all counters that cannot transfer to non-zero counts
|
||||
for (; pow2 - 1 < difference && i < exp.length () - 1; ++i)
|
||||
{
|
||||
sum -= exp[0];
|
||||
exp[0] = exp[i];
|
||||
exp[i] = 0;
|
||||
pow2 = pow2 << 1;
|
||||
}
|
||||
// if there are no more iterations we do not care
|
||||
if (hist_c->sum == 0 && !hist_c->mod)
|
||||
{
|
||||
va_heap::release (hist_c->lin);
|
||||
va_heap::release (hist_c->exp);
|
||||
ggc_free (hist_c);
|
||||
hist_c = NULL;
|
||||
return;
|
||||
}
|
||||
// we want to change index 1/(1<<portion) of iterations
|
||||
int portion = 1;
|
||||
for (; i < exp.length () - 1 && portion < 10; ++i)
|
||||
{
|
||||
// we take a sample point and suppose by uniform distribution that all
|
||||
// lesser iterations move same as this point
|
||||
gcov_type_unsigned point = (pow2 >> 1) + (pow2 >> (1 + portion));
|
||||
unsigned int ind
|
||||
= point >= difference
|
||||
? hist_index (point - difference, lin_size, exp_size) : 0;
|
||||
while (ind == i)
|
||||
{
|
||||
// if nothing changes we decrease the portion of iteration changed
|
||||
++portion;
|
||||
point = (pow2 >> 1) + (pow2 >> (1 + portion));
|
||||
ind = hist_index (point >= difference ? point - difference : 0, lin_size, exp_size);
|
||||
}
|
||||
int64_t diff = exp[i] / (1 << portion);
|
||||
exp[ind > lin_size ? ind - lin_size : 0] += diff;
|
||||
exp[i] -= diff;
|
||||
pow2 <<= 1;
|
||||
}
|
||||
if (hist_c->mod)
|
||||
{
|
||||
auto &mod_h = *(hist_c->mod);
|
||||
unsigned int mod_size = mod_h.length ();
|
||||
// rotate the values by the difference
|
||||
auto_vec<gcov_type> temp;
|
||||
for (unsigned int j = 0; j < mod_size; ++j)
|
||||
{
|
||||
temp.safe_push (mod_h[j]);
|
||||
}
|
||||
for (unsigned int j = 0; j < mod_size; ++j)
|
||||
{
|
||||
mod_h[j] = temp[(mod_size + j - (difference % mod_size)) % mod_size];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
histogram_counters_div_upper_bound (histogram_counters *&hist_c,
|
||||
unsigned int divisor)
|
||||
{
|
||||
if (divisor < 2 || !hist_c)
|
||||
return;
|
||||
if (hist_c->sum == 0)
|
||||
{
|
||||
va_heap::release (hist_c->lin);
|
||||
va_heap::release (hist_c->exp);
|
||||
if (hist_c->mod)
|
||||
va_heap::release (hist_c->mod);
|
||||
ggc_free (hist_c);
|
||||
hist_c = NULL;
|
||||
return;
|
||||
}
|
||||
auto &lin = *(hist_c->lin);
|
||||
auto &exp = *(hist_c->exp);
|
||||
hist_c->adjusted = true;
|
||||
unsigned int lin_size = lin.length ();
|
||||
unsigned int exp_size = exp.length ();
|
||||
unsigned int tot_size = lin_size + exp_size;
|
||||
unsigned int i = 1;
|
||||
for (; i < lin.length (); i++)
|
||||
{
|
||||
// we want to take the roof of the division
|
||||
lin[i / divisor + MIN (1, i % divisor)] += lin[i];
|
||||
lin[i] = 0;
|
||||
}
|
||||
lin.truncate (lin.length () ? (lin.length () - 1) / divisor
|
||||
+ MIN (1, (lin.length () - 1) % divisor) + 1
|
||||
: 0);
|
||||
|
||||
for (; i < tot_size - 1; i++)
|
||||
{
|
||||
gcov_type_unsigned upper_pow2
|
||||
= ((gcov_type_unsigned) 1) << (ceil_log2 (lin_size) + i + 1 - lin_size);
|
||||
gcov_type_unsigned half = (upper_pow2 >> 2) + (upper_pow2 >> 1);
|
||||
unsigned int ind = hist_index (half / divisor, lin_size, exp_size);
|
||||
// hist is allways different then i since we know divisor>1
|
||||
exp[ind > lin_size ? ind - lin_size : 0] += exp[i];
|
||||
exp[i] = 0;
|
||||
}
|
||||
// we do not try to maintain modulo info when dividing since it is often
|
||||
// impossible
|
||||
// TODO we can sometimes save this if the divisor and the mod_size are
|
||||
// co-prime
|
||||
if (hist_c->mod)
|
||||
{
|
||||
va_heap::release (hist_c->mod);
|
||||
hist_c->mod = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// adjust estimates after peeling npeel times
|
||||
// also used in copy header optimization
|
||||
void
|
||||
adjust_loop_estimates_minus (class loop *loop, unsigned HOST_WIDE_INT npeel,
|
||||
bool precise)
|
||||
{
|
||||
if (loop->any_estimate)
|
||||
{
|
||||
if (wi::leu_p (npeel, loop->nb_iterations_estimate))
|
||||
loop->nb_iterations_estimate -= npeel;
|
||||
else
|
||||
loop->any_estimate = false;
|
||||
}
|
||||
if (loop->any_upper_bound && precise)
|
||||
{
|
||||
if (wi::ltu_p (npeel, loop->nb_iterations_upper_bound))
|
||||
loop->nb_iterations_upper_bound -= npeel;
|
||||
else
|
||||
loop->nb_iterations_upper_bound = 0;
|
||||
}
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_minus_upper_bound (loop->counters, npeel);
|
||||
}
|
||||
if (loop->any_likely_upper_bound)
|
||||
{
|
||||
if (wi::ltu_p (npeel, loop->nb_iterations_likely_upper_bound))
|
||||
loop->nb_iterations_likely_upper_bound -= npeel;
|
||||
else
|
||||
{
|
||||
loop->any_estimate = true;
|
||||
loop->nb_iterations_estimate = 0;
|
||||
loop->nb_iterations_likely_upper_bound = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,6 +92,17 @@ struct loop_exit_hasher : ggc_ptr_hash<loop_exit>
|
||||
static void remove (loop_exit *);
|
||||
};
|
||||
|
||||
// Information about histogram of the loop from profiling
|
||||
|
||||
struct GTY (()) histogram_counters
|
||||
{
|
||||
bool adjusted;
|
||||
gcov_type sum;
|
||||
vec<gcov_type, va_heap, vl_embed> *GTY ((skip)) lin;
|
||||
vec<gcov_type, va_heap, vl_embed> *GTY ((skip)) exp;
|
||||
vec<gcov_type, va_heap, vl_embed> *GTY ((skip)) mod;
|
||||
};
|
||||
|
||||
typedef class loop *loop_p;
|
||||
|
||||
/* An integer estimation of the number of iterations. Estimate_state
|
||||
@@ -113,6 +124,8 @@ struct GTY ((chain_next ("%h.next"))) control_iv {
|
||||
struct control_iv *next;
|
||||
};
|
||||
|
||||
#include "value-prof.h"
|
||||
|
||||
/* Structure to hold information for each natural loop. */
|
||||
class GTY ((chain_next ("%h.next"))) loop {
|
||||
public:
|
||||
@@ -272,6 +285,8 @@ public:
|
||||
the basic-block from being collected but its index can still be
|
||||
reused. */
|
||||
basic_block former_header;
|
||||
|
||||
histogram_counters *counters;
|
||||
};
|
||||
|
||||
/* Set if the loop is known to be infinite. */
|
||||
@@ -916,6 +931,15 @@ extern bool get_estimated_loop_iterations (class loop *loop, widest_int *nit);
|
||||
extern bool get_max_loop_iterations (const class loop *loop, widest_int *nit);
|
||||
extern bool get_likely_max_loop_iterations (class loop *loop, widest_int *nit);
|
||||
extern int bb_loop_depth (const_basic_block);
|
||||
extern void
|
||||
histogram_counters_minus_upper_bound (histogram_counters *&hist_c,
|
||||
gcov_type_unsigned difference);
|
||||
extern void
|
||||
histogram_counters_div_upper_bound (histogram_counters *&hist_c,
|
||||
unsigned int divisor);
|
||||
extern void
|
||||
adjust_loop_estimates_minus (class loop *loop, unsigned HOST_WIDE_INT npeel,
|
||||
bool precise);
|
||||
|
||||
/* Converts VAL to widest_int. */
|
||||
|
||||
|
||||
+48
-5
@@ -959,6 +959,16 @@ copy_loop_info (class loop *loop, class loop *target)
|
||||
target->finite_p = loop->finite_p;
|
||||
target->unroll = loop->unroll;
|
||||
target->owned_clique = loop->owned_clique;
|
||||
if (loop->counters)
|
||||
{
|
||||
target->counters = ggc_alloc<histogram_counters> ();
|
||||
target->counters->sum = loop->counters->sum;
|
||||
target->counters->adjusted = loop->counters->adjusted;
|
||||
target->counters->lin = vec_safe_copy (loop->counters->lin);
|
||||
target->counters->exp = vec_safe_copy (loop->counters->exp);
|
||||
if (loop->counters->mod)
|
||||
target->counters->mod = vec_safe_copy (loop->counters->mod);
|
||||
}
|
||||
}
|
||||
|
||||
/* Copies copy of LOOP as subloop of TARGET loop, placing newly
|
||||
@@ -1148,11 +1158,44 @@ duplicate_loop_body_to_header_edge (class loop *loop, edge e,
|
||||
}
|
||||
|
||||
scale_step = XNEWVEC (profile_probability, ndupl);
|
||||
|
||||
for (i = 1; i <= ndupl; i++)
|
||||
scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
|
||||
? prob_pass_wont_exit
|
||||
: prob_pass_thru;
|
||||
// If we have histogram then we know precise probabilities for peeling
|
||||
// is_latch means unrolling
|
||||
if (!loop->counters || is_latch)
|
||||
{
|
||||
for (i = 1; i <= ndupl; i++)
|
||||
scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
|
||||
? prob_pass_wont_exit
|
||||
: prob_pass_thru;
|
||||
}
|
||||
else
|
||||
{
|
||||
gcov_type psum = 0;
|
||||
gcov_type sum = loop->counters->sum;
|
||||
for (i = 0;
|
||||
i < ndupl && i < (unsigned int) loop->counters->lin->length ()
|
||||
&& sum != psum;
|
||||
i++)
|
||||
{
|
||||
scale_step[i] = profile_probability::always ()
|
||||
- ((*loop->counters->lin)[i]
|
||||
? profile_probability::always ()
|
||||
/ (loop->counters->sum - psum)
|
||||
* (*loop->counters->lin)[i]
|
||||
: profile_probability::never ());
|
||||
// we set whether it was adjusted or is still precise
|
||||
if (loop->counters->adjusted)
|
||||
{
|
||||
scale_step[i].adjusted ();
|
||||
}
|
||||
psum += (*loop->counters->lin)[i];
|
||||
}
|
||||
++i;
|
||||
for (; i <= ndupl; i++)
|
||||
scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
|
||||
? prob_pass_wont_exit
|
||||
: prob_pass_thru;
|
||||
// should rescale
|
||||
}
|
||||
|
||||
/* Complete peeling is special as the probability of exit in last
|
||||
copy becomes 1. */
|
||||
|
||||
@@ -59,5 +59,6 @@ class loop * loop_version (class loop *, void *,
|
||||
basic_block *,
|
||||
profile_probability, profile_probability,
|
||||
profile_probability, profile_probability, bool);
|
||||
void adjust_loop_info_after_peeling (class loop *loop, int npeel, bool precise);
|
||||
|
||||
#endif /* GCC_CFGLOOPMANIP_H */
|
||||
|
||||
@@ -2291,6 +2291,14 @@ fpeel-loops
|
||||
Common Var(flag_peel_loops) Optimization
|
||||
Perform loop peeling.
|
||||
|
||||
fuse-histograms-in-peeling
|
||||
Common Var(flag_use_histogram_in_peeling) Init(1) Optimization
|
||||
Use histograms in loop peeling heuristics.
|
||||
|
||||
fpeel-loops-without-histogram
|
||||
Common Var(flag_peel_loops_without_histogram) Init(1) Optimization
|
||||
Peel also loop with no loop histogram.
|
||||
|
||||
fpeephole
|
||||
Common Var(flag_no_peephole,0) Optimization
|
||||
Enable machine specific peephole optimizations.
|
||||
@@ -2388,6 +2396,10 @@ fprofile-reproducible=
|
||||
Common Joined RejectNegative Var(flag_profile_reproducible) Enum(profile_reproducibility) Init(PROFILE_REPRODUCIBILITY_SERIAL)
|
||||
-fprofile-reproducible=[serial|parallel-runs|multithreaded] Control level of reproducibility of profile gathered by -fprofile-generate.
|
||||
|
||||
fprofile-loops
|
||||
Common Var(flag_profile_loops) Init(1) Optimization
|
||||
Profile loop histograms.
|
||||
|
||||
Enum
|
||||
Name(profile_update) Type(enum profile_update) UnknownError(unknown profile update method %qs)
|
||||
|
||||
@@ -3099,6 +3111,10 @@ fversion-loops-for-strides
|
||||
Common Var(flag_version_loops_for_strides) Optimization
|
||||
Version loops based on whether indices have a stride of one.
|
||||
|
||||
fversion-loops-using-histograms
|
||||
Common Var(flag_version_loops_using_histograms) Optimization
|
||||
Version loops based on iteration count histograms
|
||||
|
||||
funwind-tables
|
||||
Common Var(flag_unwind_tables) Optimization
|
||||
Just generate unwind tables for exception handling.
|
||||
|
||||
@@ -49,3 +49,7 @@ DEF_GCOV_COUNTER(GCOV_COUNTER_IOR, "ior", _ior)
|
||||
|
||||
/* Time profile collecting first run of a function */
|
||||
DEF_GCOV_COUNTER(GCOV_TIME_PROFILER, "time_profiler", _time_profile)
|
||||
|
||||
/* Histogram of values from 1 to 8 and them powers of 2 */
|
||||
DEF_GCOV_COUNTER(GCOV_COUNTER_V_HISTOGRAM, "histogram", _add)
|
||||
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
/* Loop versioning using histograms.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
Contributed by Ondrej Kubanek and Jan Hubicka
|
||||
|
||||
This file is part of GCC.
|
||||
|
||||
GCC is free software; you can redistribute it and/or modify it
|
||||
under the terms of the GNU General Public License as published by the
|
||||
Free Software Foundation; either version 3, or (at your option) any
|
||||
later version.
|
||||
|
||||
GCC is distributed in the hope that it will be useful, but WITHOUT
|
||||
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
||||
for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with GCC; see the file COPYING3. If not see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include "config.h"
|
||||
#include "system.h"
|
||||
#include "coretypes.h"
|
||||
#include "backend.h"
|
||||
#include "tree.h"
|
||||
#include "gimple.h"
|
||||
#include "gimple-iterator.h"
|
||||
#include "tree-pass.h"
|
||||
#include "gimplify-me.h"
|
||||
#include "cfgloop.h"
|
||||
#include "tree-ssa-loop.h"
|
||||
#include "ssa.h"
|
||||
#include "tree-scalar-evolution.h"
|
||||
#include "tree-ssa-loop-ivopts.h"
|
||||
#include "fold-const.h"
|
||||
#include "tree-ssa-propagate.h"
|
||||
#include "tree-inline.h"
|
||||
#include "domwalk.h"
|
||||
#include "tree-vectorizer.h"
|
||||
#include "omp-general.h"
|
||||
#include "predict.h"
|
||||
#include "tree-into-ssa.h"
|
||||
#include "gimple-range.h"
|
||||
#include "tree-cfg.h"
|
||||
#include "tree-ssa-loop-niter.h"
|
||||
#include "tree-pretty-print.h"
|
||||
|
||||
namespace {
|
||||
|
||||
/* Histogram loop versioning pass.
|
||||
The pass looks for loops with single exit conditioned by a induction
|
||||
variable test. If histogram indicates that such loop iterates usually
|
||||
N times, we produce a versined version:
|
||||
|
||||
if (number_of_iterations == N)
|
||||
for (int i = 0; i < N; i++)
|
||||
loop_body;
|
||||
else
|
||||
original_loop.
|
||||
After this transformation the specialized version can often be
|
||||
fully unrolled or vectorized. */
|
||||
|
||||
const pass_data pass_data_loop_histogram_versioning =
|
||||
{
|
||||
GIMPLE_PASS, /* type */
|
||||
"hversion", /* name */
|
||||
OPTGROUP_LOOP, /* optinfo_flags */
|
||||
TV_LOOP_VERSIONING, /* tv_id */
|
||||
PROP_cfg, /* properties_required */
|
||||
0, /* properties_provided */
|
||||
0, /* properties_destroyed */
|
||||
0, /* todo_flags_start */
|
||||
0, /* todo_flags_finish */
|
||||
};
|
||||
|
||||
class pass_loop_histogram_versioning : public gimple_opt_pass
|
||||
{
|
||||
public:
|
||||
pass_loop_histogram_versioning (gcc::context *ctxt)
|
||||
: gimple_opt_pass (pass_data_loop_histogram_versioning, ctxt)
|
||||
{}
|
||||
|
||||
/* opt_pass methods: */
|
||||
bool gate (function *) final override
|
||||
{
|
||||
return flag_version_loops_using_histograms;
|
||||
}
|
||||
unsigned int execute (function *) final override;
|
||||
};
|
||||
|
||||
unsigned int
|
||||
pass_loop_histogram_versioning::execute (function *fn)
|
||||
{
|
||||
bool changed = false;
|
||||
if (number_of_loops (fn) <= 1)
|
||||
return 0;
|
||||
loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
|
||||
scev_initialize ();
|
||||
estimate_numbers_of_iterations (cfun);
|
||||
|
||||
for (auto loop : loops_list (cfun, 0))
|
||||
{
|
||||
/* We version only inner loops optimized for speed with histogram
|
||||
attachd. */
|
||||
if (loop->inner != NULL
|
||||
|| loop->counters == NULL
|
||||
|| !optimize_loop_for_speed_p (loop))
|
||||
continue;
|
||||
int best_iters = - 1;
|
||||
gcov_type best_val = 0;
|
||||
|
||||
/* Look for dominating iteration count. */
|
||||
for (int i = 0; i < loop->counters->lin->length (); i++)
|
||||
{
|
||||
if ((*(loop->counters->lin))[i] > best_val)
|
||||
{
|
||||
best_val = (*(loop->counters->lin))[i];
|
||||
best_iters = i;
|
||||
}
|
||||
}
|
||||
if (best_val <= 0 || best_val < loop->counters->sum * param_loop_versioning_histogram_prcnt / 100)
|
||||
continue;
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Loop %i has dominating number of iterations %i\n",
|
||||
loop->num, best_iters);
|
||||
if (!best_iters)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file,
|
||||
"Not versioning since normal peeling will do the job\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
/* If loop has no single exit, don't try to version.
|
||||
In this case we don not know if the loop iteration count corresponds
|
||||
to the IV test.
|
||||
TODO: We may be able to also handle loop with single dominating
|
||||
exit. */
|
||||
auto_vec<edge> exits = get_loop_exit_edges (loop);
|
||||
if (exits.length () != 1)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Wrong number of exits: %i\n", exits.length ());
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Analyze number of iterations. */
|
||||
class tree_niter_desc niter;
|
||||
if (!number_of_iterations_exit (loop, exits[0],
|
||||
&niter, false, false))
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "niter analysis failed\n");
|
||||
continue;
|
||||
}
|
||||
/* If the loop exit is not simple IV test or the loop already
|
||||
has known number of iterations, give up. */
|
||||
if (!niter.niter
|
||||
|| !integer_onep (niter.assumptions)
|
||||
|| TREE_CODE (niter.niter) == INTEGER_CST)
|
||||
{
|
||||
if (dump_file)
|
||||
{
|
||||
if (TREE_CODE (niter.niter) == INTEGER_CST)
|
||||
fprintf (dump_file, "niter is already constant:");
|
||||
else
|
||||
fprintf (dump_file, "niter wrong:");
|
||||
print_generic_expr (dump_file, niter.niter);
|
||||
fprintf (dump_file, "\n");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Sanity check with maximal number of iterations. */
|
||||
int maxiter = max_loop_iterations_int (loop);
|
||||
if (maxiter >= 0 && maxiter < best_iters)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Upper bound on number of iterations is too small\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Compute loop body size. */
|
||||
int ninsns = 0;
|
||||
basic_block *bbs = get_loop_body (loop);
|
||||
for (unsigned int i = 0; i < loop->num_nodes; i++)
|
||||
for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
|
||||
!gsi_end_p (gsi) && ninsns <= param_loop_versioning_max_inner_insns;
|
||||
gsi_next (&gsi))
|
||||
ninsns += estimate_num_insns (gsi_stmt (gsi), &eni_size_weights);
|
||||
free (bbs);
|
||||
if (ninsns > param_loop_versioning_max_inner_insns)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "--param loop-versioning-max-inner-insns exceeded\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
/* Do versining. */
|
||||
dump_user_location_t locus = last_stmt (exits[0]->src);
|
||||
basic_block cond_bb;
|
||||
gimple_seq stmts = NULL;
|
||||
initialize_original_copy_tables ();
|
||||
tree op = force_gimple_operand_1
|
||||
(fold_build2 (NE_EXPR, boolean_type_node,
|
||||
niter.niter,
|
||||
build_int_cst (TREE_TYPE (niter.niter),
|
||||
best_iters)),
|
||||
&stmts, is_gimple_condexpr_for_cond, NULL_TREE);
|
||||
|
||||
class loop *optimized_loop
|
||||
= loop_version (loop, op, &cond_bb,
|
||||
/* It may be more precise to take probability from
|
||||
histogram, but it also may lead for
|
||||
over-specialization for the train run. */
|
||||
profile_probability::very_unlikely (),
|
||||
profile_probability::very_likely (),
|
||||
profile_probability::very_unlikely (),
|
||||
profile_probability::very_likely (), true);
|
||||
free_original_copy_tables ();
|
||||
|
||||
/* Insert IV test. */
|
||||
if (stmts)
|
||||
{
|
||||
gimple_stmt_iterator gsi = gsi_last_bb (cond_bb);
|
||||
gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
|
||||
}
|
||||
|
||||
/* Update optimized loop.
|
||||
We force it to have given number of iteraitons and thus
|
||||
it makes sense to drop histogram counters. */
|
||||
record_niter_bound (optimized_loop, best_val, false, true);
|
||||
optimized_loop->counters = 0;
|
||||
auto_vec<edge> optimized_exits = get_loop_exit_edges (optimized_loop);
|
||||
create_canonical_iv (optimized_loop, optimized_exits[0],
|
||||
build_int_cst (integer_type_node, best_iters));
|
||||
/* Update counters of non-optiized loop.
|
||||
TODO: also recompute estimated number of itraitons
|
||||
instead of droping it bellow. */
|
||||
(*(loop->counters->lin))[best_iters] = 0;
|
||||
loop->counters->sum -= best_val;
|
||||
loop->any_estimate = false;
|
||||
changed = true;
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | TDF_DETAILS, locus,
|
||||
"loop %i versioned based on histogram "
|
||||
"for %i iterations (header execution count %d)\n",
|
||||
loop->num, best_iters,
|
||||
optimized_loop->header->count.to_gcov_type ());
|
||||
}
|
||||
free_numbers_of_iterations_estimates (cfun);
|
||||
scev_finalize ();
|
||||
loop_optimizer_finalize ();
|
||||
return (changed ? TODO_update_ssa : 0) | TODO_cleanup_cfg;
|
||||
}
|
||||
|
||||
} // anon namespace
|
||||
|
||||
gimple_opt_pass *
|
||||
make_pass_loop_histogram_versioning (gcc::context *ctxt)
|
||||
{
|
||||
return new pass_loop_histogram_versioning (ctxt);
|
||||
}
|
||||
@@ -41,6 +41,8 @@ along with GCC; see the file COPYING3. If not see
|
||||
#include "tree-into-ssa.h"
|
||||
#include "gimple-range.h"
|
||||
#include "tree-cfg.h"
|
||||
#include "tree-ssa-loop-niter.h"
|
||||
#include "tree-pretty-print.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -1808,3 +1810,4 @@ make_pass_loop_versioning (gcc::context *ctxt)
|
||||
{
|
||||
return new pass_loop_versioning (ctxt);
|
||||
}
|
||||
|
||||
|
||||
@@ -533,6 +533,10 @@ unroll_loop_constant_iterations (class loop *loop)
|
||||
desc->noloop_assumptions = NULL_RTX;
|
||||
desc->niter -= exit_mod;
|
||||
loop->nb_iterations_upper_bound -= exit_mod;
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_minus_upper_bound (loop->counters, exit_mod);
|
||||
}
|
||||
if (loop->any_estimate
|
||||
&& wi::leu_p (exit_mod, loop->nb_iterations_estimate))
|
||||
loop->nb_iterations_estimate -= exit_mod;
|
||||
@@ -578,6 +582,11 @@ unroll_loop_constant_iterations (class loop *loop)
|
||||
|
||||
desc->niter -= exit_mod + 1;
|
||||
loop->nb_iterations_upper_bound -= exit_mod + 1;
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_minus_upper_bound (loop->counters,
|
||||
exit_mod + 1);
|
||||
}
|
||||
if (loop->any_estimate
|
||||
&& wi::leu_p (exit_mod + 1, loop->nb_iterations_estimate))
|
||||
loop->nb_iterations_estimate -= exit_mod + 1;
|
||||
@@ -632,6 +641,10 @@ unroll_loop_constant_iterations (class loop *loop)
|
||||
desc->niter /= max_unroll + 1;
|
||||
loop->nb_iterations_upper_bound
|
||||
= wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_div_upper_bound (loop->counters, exit_mod);
|
||||
}
|
||||
if (loop->any_estimate)
|
||||
loop->nb_iterations_estimate
|
||||
= wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);
|
||||
@@ -1095,6 +1108,10 @@ unroll_loop_runtime_iterations (class loop *loop)
|
||||
gen_int_mode (max_unroll + 1, desc->mode));
|
||||
loop->nb_iterations_upper_bound
|
||||
= wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_div_upper_bound (loop->counters, max_unroll + 1);
|
||||
}
|
||||
if (loop->any_estimate)
|
||||
loop->nb_iterations_estimate
|
||||
= wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);
|
||||
@@ -1107,6 +1124,10 @@ unroll_loop_runtime_iterations (class loop *loop)
|
||||
simplify_gen_binary (MINUS, desc->mode, desc->niter_expr, const1_rtx);
|
||||
desc->noloop_assumptions = NULL_RTX;
|
||||
--loop->nb_iterations_upper_bound;
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_minus_upper_bound (loop->counters, 1);
|
||||
}
|
||||
if (loop->any_estimate
|
||||
&& loop->nb_iterations_estimate != 0)
|
||||
--loop->nb_iterations_estimate;
|
||||
|
||||
@@ -1126,6 +1126,38 @@ input_cfg (class lto_input_block *ib, class data_in *data_in,
|
||||
loop->any_estimate = streamer_read_hwi (ib);
|
||||
if (loop->any_estimate)
|
||||
loop->nb_iterations_estimate = streamer_read_widest_int (ib);
|
||||
// whether the loop has histogram counter
|
||||
if (streamer_read_hwi (ib))
|
||||
{
|
||||
loop->counters = ggc_alloc<histogram_counters> ();
|
||||
loop->counters->sum = streamer_read_gcov_count (ib);
|
||||
loop->counters->lin = NULL;
|
||||
loop->counters->exp = NULL;
|
||||
loop->counters->mod = NULL;
|
||||
int lin_len = streamer_read_hwi (ib);
|
||||
int exp_len = streamer_read_hwi (ib);
|
||||
int mod_len = streamer_read_hwi (ib);
|
||||
vec_safe_grow (loop->counters->lin, lin_len);
|
||||
vec_safe_grow (loop->counters->exp, exp_len);
|
||||
for (int i = 0; i < lin_len; ++i)
|
||||
{
|
||||
(*loop->counters->lin)[i] = streamer_read_gcov_count (ib);
|
||||
}
|
||||
for (int i = 0; i < exp_len; ++i)
|
||||
{
|
||||
(*loop->counters->exp)[i] = streamer_read_gcov_count (ib);
|
||||
}
|
||||
// do we have modulos
|
||||
if (mod_len)
|
||||
{
|
||||
vec_safe_grow (loop->counters->mod,
|
||||
mod_len);
|
||||
for (int i = 0; i < mod_len; ++i)
|
||||
{
|
||||
(*loop->counters->mod)[i] = streamer_read_gcov_count (ib);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Read OMP SIMD related info. */
|
||||
loop->safelen = streamer_read_hwi (ib);
|
||||
|
||||
@@ -2181,6 +2181,31 @@ output_cfg (struct output_block *ob, struct function *fn)
|
||||
streamer_write_hwi (ob, loop->any_estimate);
|
||||
if (loop->any_estimate)
|
||||
streamer_write_widest_int (ob, loop->nb_iterations_estimate);
|
||||
streamer_write_hwi (ob, loop->counters != NULL);
|
||||
if (loop->counters)
|
||||
{
|
||||
streamer_write_gcov_count (ob, loop->counters->sum);
|
||||
streamer_write_hwi (ob, loop->counters->lin->length ());
|
||||
streamer_write_hwi (ob, loop->counters->exp->length ());
|
||||
streamer_write_hwi (ob, loop->counters->mod ? loop->counters->mod->length () : 0);
|
||||
for (unsigned int i = 0; i < loop->counters->lin->length (); ++i)
|
||||
{
|
||||
streamer_write_gcov_count (ob, (*loop->counters->lin)[i]);
|
||||
}
|
||||
for (unsigned int i = 0;
|
||||
i < (unsigned int) loop->counters->exp->length (); ++i)
|
||||
{
|
||||
streamer_write_gcov_count (ob, (*loop->counters->exp)[i]);
|
||||
}
|
||||
if (loop->counters->mod)
|
||||
{
|
||||
for (unsigned int i = 0;
|
||||
i < (unsigned int) loop->counters->mod->length (); ++i)
|
||||
{
|
||||
streamer_write_gcov_count (ob, (*loop->counters->mod)[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Write OMP SIMD related info. */
|
||||
streamer_write_hwi (ob, loop->safelen);
|
||||
|
||||
@@ -2044,6 +2044,7 @@ enable_fdo_optimizations (struct gcc_options *opts,
|
||||
SET_OPTION_IF_UNSET (opts, opts_set, flag_loop_interchange, value);
|
||||
SET_OPTION_IF_UNSET (opts, opts_set, flag_unroll_jam, value);
|
||||
SET_OPTION_IF_UNSET (opts, opts_set, flag_tree_loop_distribution, value);
|
||||
SET_OPTION_IF_UNSET (opts, opts_set, flag_version_loops_using_histograms, value);
|
||||
}
|
||||
|
||||
/* -f{,no-}sanitize{,-recover}= suboptions. */
|
||||
|
||||
@@ -426,6 +426,10 @@ The maximum number of instructions in an inner loop that is being considered for
|
||||
Common Joined UInteger Var(param_loop_versioning_max_outer_insns) Init(100) Param Optimization
|
||||
The maximum number of instructions in an outer loop that is being considered for versioning, on top of the instructions in inner loops.
|
||||
|
||||
-param=loop-versioning-histogram_prcnt=
|
||||
Common Joined UInteger Var(param_loop_versioning_histogram_prcnt) Init(30) Param Optimization
|
||||
Percentage of the dominating iteration count required for versioning.
|
||||
|
||||
-param=lra-inheritance-ebb-probability-cutoff=
|
||||
Common Joined UInteger Var(param_lra_inheritance_ebb_probability_cutoff) Init(40) IntegerRange(0, 100) Param Optimization
|
||||
Minimal fall-through edge probability in percentage used to add BB to inheritance EBB in LRA.
|
||||
@@ -874,6 +878,26 @@ Min. ratio of insns to mem ops to enable prefetching in a loop.
|
||||
Common Joined UInteger Var(param_prefetch_minimum_stride) Init(-1) Param Optimization
|
||||
The minimum constant stride beyond which we should use prefetch hints for.
|
||||
|
||||
-param=profile-histogram-size-exp=
|
||||
Common Joined UInteger Var(param_profile_histogram_size_exp) Init(1) IntegerRange(1, 255) Param Optimization
|
||||
Size of additional counters in exponential intervals after the linear section.
|
||||
|
||||
-param=profile-histogram-size-mod=
|
||||
Common Joined UInteger Var(param_profile_histogram_size_mod) Init(32) IntegerRange(1, 255) Param Optimization
|
||||
Counts the modulo of loop iterations by its size.
|
||||
|
||||
-param=profile-histogram-size-lin=
|
||||
Common Joined UInteger Var(param_profile_histogram_size_lin) Init(16) IntegerRange(1, 255) Param Optimization
|
||||
Size of the linear portion of the histogram counter for profile feedback.
|
||||
|
||||
-param=profile-histogram-peel-prcnt=
|
||||
Common Joined UInteger Var(param_profile_histogram_peel_prcnt) Init(6) Param Optimization
|
||||
Peels if it can eliminate at least this percentage per 1 peeling.
|
||||
|
||||
-param=profile-histogram-peel-overall-prcnt=
|
||||
Common Joined UInteger Var(param_profile_histogram_peel_overall_prcnt) Init(30) Param Optimization
|
||||
Peels if it can eliminate at least this percentage of executions of the loop.
|
||||
|
||||
-param=profile-func-internal-id=
|
||||
Common Joined UInteger Var(param_profile_func_internal_id) IntegerRange(0, 1) Param
|
||||
Use internal function id in profile lookup.
|
||||
|
||||
@@ -229,6 +229,7 @@ along with GCC; see the file COPYING3. If not see
|
||||
NEXT_PASS (pass_merge_phi);
|
||||
NEXT_PASS (pass_phiopt, false /* early_p */);
|
||||
NEXT_PASS (pass_tail_recursion);
|
||||
NEXT_PASS (pass_loop_histogram_versioning);
|
||||
NEXT_PASS (pass_ch);
|
||||
NEXT_PASS (pass_lower_complex);
|
||||
NEXT_PASS (pass_sra);
|
||||
@@ -275,6 +276,7 @@ along with GCC; see the file COPYING3. If not see
|
||||
/* Before loop_init we rewrite no longer addressed locals into SSA
|
||||
form if possible. */
|
||||
NEXT_PASS (pass_tree_loop_init);
|
||||
//NEXT_PASS (pass_loop_histogram_versioning);
|
||||
NEXT_PASS (pass_tree_unswitch);
|
||||
NEXT_PASS (pass_scev_cprop);
|
||||
NEXT_PASS (pass_loop_split);
|
||||
|
||||
+70
-4
@@ -66,6 +66,7 @@ along with GCC; see the file COPYING3. If not see
|
||||
#include "cfgloop.h"
|
||||
#include "sreal.h"
|
||||
#include "file-prefix-map.h"
|
||||
#include "tree-scalar-evolution.h"
|
||||
|
||||
#include "profile.h"
|
||||
|
||||
@@ -189,6 +190,10 @@ instrument_values (histogram_values values)
|
||||
gimple_gen_time_profiler (t);
|
||||
break;
|
||||
|
||||
case HIST_TYPE_HISTOGRAM:
|
||||
gimple_gen_histogram_profiler (hist, t);
|
||||
break;
|
||||
|
||||
default:
|
||||
gcc_unreachable ();
|
||||
}
|
||||
@@ -915,8 +920,63 @@ compute_value_histograms (histogram_values values, unsigned cfg_checksum,
|
||||
bool topn_p = (hist->type == HIST_TYPE_TOPN_VALUES
|
||||
|| hist->type == HIST_TYPE_INDIR_CALL);
|
||||
|
||||
/* Histogram profiler counter is not related to any statement,
|
||||
so that we have to read the counter and set the value to
|
||||
the corresponding loop */
|
||||
|
||||
if (hist->type == HIST_TYPE_HISTOGRAM)
|
||||
{
|
||||
auto lp = hist->hvalue.lp;
|
||||
if (act_count[t])
|
||||
{
|
||||
lp->counters = ggc_alloc<histogram_counters> ();
|
||||
gcov_type sum = 0;
|
||||
unsigned int tot_size = param_profile_histogram_size_lin
|
||||
+ param_profile_histogram_size_exp;
|
||||
lp->counters->adjusted = false;
|
||||
lp->counters->lin = NULL;
|
||||
lp->counters->exp = NULL;
|
||||
// iteration vector
|
||||
vec_safe_grow_cleared (lp->counters->lin,
|
||||
param_profile_histogram_size_lin);
|
||||
vec_safe_grow_cleared (lp->counters->exp,
|
||||
param_profile_histogram_size_exp);
|
||||
for (int i = 0; i < (int) param_profile_histogram_size_lin; ++i)
|
||||
{
|
||||
auto hst = lp->counters->lin;
|
||||
(*hst)[i] = act_count[t][i];
|
||||
sum += act_count[t][i];
|
||||
}
|
||||
for (int i = 0; i < (int) param_profile_histogram_size_exp; ++i)
|
||||
{
|
||||
auto hst = lp->counters->exp;
|
||||
(*hst)[i] = act_count[t][i + param_profile_histogram_size_lin];
|
||||
sum += act_count[t][i + param_profile_histogram_size_lin];
|
||||
}
|
||||
lp->counters->sum = sum;
|
||||
|
||||
// modular vector
|
||||
lp->counters->mod = NULL;
|
||||
vec_safe_grow_cleared (lp->counters->mod,
|
||||
param_profile_histogram_size_mod);
|
||||
for (int j = 0; j < param_profile_histogram_size_mod; ++j)
|
||||
{
|
||||
auto mod = lp->counters->mod;
|
||||
(*mod)[j] = act_count[t][j + tot_size];
|
||||
}
|
||||
if (sum == 0)
|
||||
{
|
||||
va_heap::release (lp->counters->lin);
|
||||
va_heap::release (lp->counters->exp);
|
||||
va_heap::release (lp->counters->mod);
|
||||
ggc_free (lp->counters);
|
||||
lp->counters = NULL;
|
||||
}
|
||||
act_count[t] += hist->n_counters;
|
||||
}
|
||||
}
|
||||
/* TOP N counter uses variable number of counters. */
|
||||
if (topn_p)
|
||||
else if (topn_p)
|
||||
{
|
||||
unsigned total_size;
|
||||
if (act_count[t])
|
||||
@@ -1176,6 +1236,10 @@ branch_prob (bool thunk)
|
||||
|
||||
total_num_times_called++;
|
||||
|
||||
calculate_dominance_info (CDI_DOMINATORS);
|
||||
loop_optimizer_init (LOOPS_NORMAL);
|
||||
scev_initialize ();
|
||||
|
||||
flow_call_edges_add (NULL);
|
||||
add_noreturn_fake_exit_edges ();
|
||||
|
||||
@@ -1528,16 +1592,16 @@ branch_prob (bool thunk)
|
||||
|
||||
if (flag_profile_values)
|
||||
instrument_values (values);
|
||||
|
||||
/* Commit changes done by instrumentation. */
|
||||
gsi_commit_edge_inserts ();
|
||||
}
|
||||
/* Commit changes done by instrumentation. */
|
||||
gsi_commit_edge_inserts ();
|
||||
|
||||
free_aux_for_edges ();
|
||||
|
||||
values.release ();
|
||||
free_edge_list (el);
|
||||
coverage_end_function (lineno_checksum, cfg_checksum);
|
||||
scev_finalize ();
|
||||
if (flag_branch_probabilities
|
||||
&& (profile_status_for_fn (cfun) == PROFILE_READ))
|
||||
{
|
||||
@@ -1556,6 +1620,8 @@ branch_prob (bool thunk)
|
||||
}
|
||||
compute_function_frequency ();
|
||||
}
|
||||
loop_optimizer_finalize ();
|
||||
free_dominance_info (CDI_DOMINATORS);
|
||||
}
|
||||
|
||||
/* Union find algorithm implementation for the basic blocks using
|
||||
|
||||
@@ -373,6 +373,7 @@ extern gimple_opt_pass *make_pass_tree_loop (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_tree_no_loop (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_tree_loop_init (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_loop_versioning (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_loop_histogram_versioning (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_lim (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_linterchange (gcc::context *ctxt);
|
||||
extern gimple_opt_pass *make_pass_tree_unswitch (gcc::context *ctxt);
|
||||
|
||||
@@ -52,16 +52,21 @@ along with GCC; see the file COPYING3. If not see
|
||||
#include "stringpool.h"
|
||||
#include "attribs.h"
|
||||
#include "tree-pretty-print.h"
|
||||
#include "gimple-pretty-print.h"
|
||||
#include "langhooks.h"
|
||||
#include "stor-layout.h"
|
||||
#include "xregex.h"
|
||||
#include "alloc-pool.h"
|
||||
#include "symbol-summary.h"
|
||||
#include "symtab-thunks.h"
|
||||
#include "cfgloop.h"
|
||||
|
||||
#include "print-tree.h"
|
||||
|
||||
static GTY(()) tree gcov_type_node;
|
||||
static GTY(()) tree tree_interval_profiler_fn;
|
||||
static GTY(()) tree tree_pow2_profiler_fn;
|
||||
static GTY (()) tree tree_histogram_profiler_fn;
|
||||
static GTY(()) tree tree_topn_values_profiler_fn;
|
||||
static GTY(()) tree tree_indirect_call_profiler_fn;
|
||||
static GTY(()) tree tree_average_profiler_fn;
|
||||
@@ -119,6 +124,7 @@ gimple_init_gcov_profiler (void)
|
||||
{
|
||||
tree interval_profiler_fn_type;
|
||||
tree pow2_profiler_fn_type;
|
||||
tree histogram_profiler_fn_type;
|
||||
tree topn_values_profiler_fn_type;
|
||||
tree gcov_type_ptr;
|
||||
tree ic_profiler_fn_type;
|
||||
@@ -161,6 +167,19 @@ gimple_init_gcov_profiler (void)
|
||||
= tree_cons (get_identifier ("leaf"), NULL,
|
||||
DECL_ATTRIBUTES (tree_pow2_profiler_fn));
|
||||
|
||||
/* void (*) (gcov_type *, gcov_type) */
|
||||
histogram_profiler_fn_type
|
||||
= build_function_type_list (void_type_node, gcov_type_ptr,
|
||||
gcov_type_node, gcov_type_node, NULL_TREE);
|
||||
fn_name = concat ("__gcov_histogram_profiler", fn_suffix, NULL);
|
||||
tree_histogram_profiler_fn
|
||||
= build_fn_decl (fn_name, histogram_profiler_fn_type);
|
||||
free (CONST_CAST (char *, fn_name));
|
||||
TREE_NOTHROW (tree_histogram_profiler_fn) = 1;
|
||||
DECL_ATTRIBUTES (tree_histogram_profiler_fn)
|
||||
= tree_cons (get_identifier ("leaf"), NULL,
|
||||
DECL_ATTRIBUTES (tree_histogram_profiler_fn));
|
||||
|
||||
/* void (*) (gcov_type *, gcov_type) */
|
||||
topn_values_profiler_fn_type
|
||||
= build_function_type_list (void_type_node,
|
||||
@@ -228,6 +247,7 @@ gimple_init_gcov_profiler (void)
|
||||
late, we need to initialize them by hand. */
|
||||
DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
|
||||
DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
|
||||
DECL_ASSEMBLER_NAME (tree_histogram_profiler_fn);
|
||||
DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
|
||||
DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
|
||||
DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
|
||||
@@ -336,6 +356,32 @@ gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
|
||||
gsi_insert_before (&gsi, call, GSI_NEW_STMT);
|
||||
}
|
||||
|
||||
void
|
||||
gimple_gen_histogram_profiler (histogram_value value,
|
||||
unsigned tag) // , edge_def* edge
|
||||
{
|
||||
gimple *stmt = value->hvalue.stmt;
|
||||
gcc_assert (!stmt);
|
||||
auto lp = value->hvalue.lp;
|
||||
gcc_assert (lp);
|
||||
tree ref_ptr = tree_coverage_counter_addr (tag, 0);
|
||||
tree hist_size = build_int_cst_type (
|
||||
gcov_type_node, param_profile_histogram_size_lin
|
||||
| (gcov_type (param_profile_histogram_size_exp) << 10)
|
||||
| (gcov_type (param_profile_histogram_size_mod) << 20));
|
||||
gcall *call;
|
||||
auto_vec<edge> exits = get_loop_exit_edges (lp);
|
||||
for (auto exit : exits)
|
||||
{
|
||||
if (!(exit->flags & (EDGE_COMPLEX | EDGE_FAKE)))
|
||||
{
|
||||
call = gimple_build_call (tree_histogram_profiler_fn, 3, ref_ptr,
|
||||
value->hvalue.value, hist_size);
|
||||
gsi_insert_seq_on_edge (exit, call);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Output instructions as GIMPLE trees for code to find the most N common
|
||||
values. VALUE is the expression whose value is profiled. TAG is the tag
|
||||
of the section for counters. */
|
||||
|
||||
+119
-12
@@ -381,7 +381,7 @@ unsigned int
|
||||
ch_base::copy_headers (function *fun)
|
||||
{
|
||||
basic_block header;
|
||||
edge exit, entry;
|
||||
edge exit, nonexit, entry;
|
||||
basic_block *bbs, *copied_bbs;
|
||||
unsigned n_bbs;
|
||||
unsigned bbs_size;
|
||||
@@ -453,8 +453,16 @@ ch_base::copy_headers (function *fun)
|
||||
the header to have just a single successor and copying up to
|
||||
postdominator. */
|
||||
|
||||
exit = NULL;
|
||||
nonexit = NULL;
|
||||
n_bbs = 0;
|
||||
int nexits = 0;
|
||||
profile_count exit_count = profile_count::zero ();
|
||||
profile_count entry_count = profile_count::zero ();
|
||||
edge e;
|
||||
edge_iterator ei;
|
||||
FOR_EACH_EDGE (e, ei, loop->header->preds)
|
||||
if (e->src != loop->latch)
|
||||
entry_count += e->count ();
|
||||
while (should_duplicate_loop_header_p (header, loop, &remaining_limit))
|
||||
{
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
@@ -463,15 +471,23 @@ ch_base::copy_headers (function *fun)
|
||||
/* Find a successor of header that is inside a loop; i.e. the new
|
||||
header after the condition is copied. */
|
||||
if (flow_bb_inside_loop_p (loop, EDGE_SUCC (header, 0)->dest))
|
||||
exit = EDGE_SUCC (header, 0);
|
||||
{
|
||||
nonexit = EDGE_SUCC (header, 0);
|
||||
exit = EDGE_SUCC (header, 1);
|
||||
}
|
||||
else
|
||||
exit = EDGE_SUCC (header, 1);
|
||||
{
|
||||
nonexit = EDGE_SUCC (header, 1);
|
||||
exit = EDGE_SUCC (header, 0);
|
||||
}
|
||||
exit_count += exit->count ();
|
||||
nexits++;
|
||||
bbs[n_bbs++] = header;
|
||||
gcc_assert (bbs_size > n_bbs);
|
||||
header = exit->dest;
|
||||
header = nonexit->dest;
|
||||
}
|
||||
|
||||
if (!exit)
|
||||
if (!nonexit)
|
||||
continue;
|
||||
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
@@ -483,8 +499,11 @@ ch_base::copy_headers (function *fun)
|
||||
|
||||
/* Ensure that the header will have just the latch as a predecessor
|
||||
inside the loop. */
|
||||
if (!single_pred_p (exit->dest))
|
||||
exit = single_pred_edge (split_edge (exit));
|
||||
if (!single_pred_p (nonexit->dest))
|
||||
{
|
||||
header = split_edge (nonexit);
|
||||
exit = single_pred_edge (header);
|
||||
}
|
||||
|
||||
entry = loop_preheader_edge (loop);
|
||||
|
||||
@@ -542,10 +561,20 @@ ch_base::copy_headers (function *fun)
|
||||
}
|
||||
}
|
||||
|
||||
/* Ensure that the latch and the preheader is simple (we know that they
|
||||
are not now, since there was the loop exit condition. */
|
||||
split_edge (loop_preheader_edge (loop));
|
||||
split_edge (loop_latch_edge (loop));
|
||||
/* Update header of the loop. */
|
||||
loop->header = header;
|
||||
/* Find correct latch. We only duplicate chain of conditionals so
|
||||
there should be precisely two edges to the new header. One entry
|
||||
edge and one to latch. */
|
||||
FOR_EACH_EDGE (e, ei, loop->header->preds)
|
||||
if (header != e->src)
|
||||
{
|
||||
loop->latch = e->src;
|
||||
break;
|
||||
}
|
||||
/* Ensure that the latch is simple. */
|
||||
if (!single_succ_p (loop_latch_edge (loop)->src))
|
||||
split_edge (loop_latch_edge (loop));
|
||||
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
{
|
||||
@@ -554,8 +583,86 @@ ch_base::copy_headers (function *fun)
|
||||
else
|
||||
fprintf (dump_file, "Loop %d is still not do-while loop.\n",
|
||||
loop->num);
|
||||
fprintf (dump_file, "Exit count: ");
|
||||
exit_count.dump (dump_file);
|
||||
fprintf (dump_file, "\nEntry count: ");
|
||||
entry_count.dump (dump_file);
|
||||
fprintf (dump_file, "\n");
|
||||
}
|
||||
|
||||
/* We possibly decreased number of itrations by 1. */
|
||||
auto_vec<edge> exits = get_loop_exit_edges (loop);
|
||||
bool precise = (nexits == (int) exits.length ());
|
||||
if (!get_max_loop_iterations_int (loop))
|
||||
{
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
fprintf (dump_file, "Loop %d no longer loops.\n", loop->num);
|
||||
/* TODO: We can unloop like in tree-ssa-loop-ivcanon. */
|
||||
precise = false;
|
||||
}
|
||||
/* Check that loop may not terminate in other way than via
|
||||
basic blocks we duplicated. */
|
||||
if (precise)
|
||||
{
|
||||
basic_block *bbs = get_loop_body (loop);
|
||||
for (unsigned i = 0; i < loop->num_nodes && precise; ++i)
|
||||
{
|
||||
basic_block bb = bbs[i];
|
||||
bool found_exit = false;
|
||||
FOR_EACH_EDGE (e, ei, bb->succs)
|
||||
if (!flow_bb_inside_loop_p (loop, e->dest))
|
||||
{
|
||||
found_exit = true;
|
||||
break;
|
||||
}
|
||||
/* If BB has exit, it was duplicated. */
|
||||
if (found_exit)
|
||||
continue;
|
||||
/* Give up on irreducible loops. */
|
||||
if (bb->flags & BB_IRREDUCIBLE_LOOP)
|
||||
{
|
||||
precise = false;
|
||||
break;
|
||||
}
|
||||
/* Check that inner loops are finite. */
|
||||
for (class loop *l = bb->loop_father; l != loop && precise;
|
||||
l = loop_outer (l))
|
||||
if (!l->finite_p)
|
||||
{
|
||||
precise = false;
|
||||
break;
|
||||
}
|
||||
/* Verify that there is no statement that may be terminate
|
||||
execution in a way not visible to CFG. */
|
||||
for (gimple_stmt_iterator bsi = gsi_start_bb (bb);
|
||||
!gsi_end_p (bsi); gsi_next (&bsi))
|
||||
if (stmt_can_terminate_bb_p (gsi_stmt (bsi)))
|
||||
precise = false;
|
||||
}
|
||||
}
|
||||
if (precise)
|
||||
{
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
fprintf (dump_file,
|
||||
"Peeled all exits:"
|
||||
" decreased number of iterations of loop %d by 1.\n",
|
||||
loop->num);
|
||||
adjust_loop_info_after_peeling (loop, 1, true);
|
||||
}
|
||||
else if (exit_count >= entry_count.apply_scale (9, 10))
|
||||
{
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
fprintf (dump_file,
|
||||
"Peeled likely exits: likely decreased number "
|
||||
"of iterations of loop %d by 1.\n", loop->num);
|
||||
adjust_loop_info_after_peeling (loop, 1, false);
|
||||
}
|
||||
else if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
fprintf (dump_file,
|
||||
"Not decreased number"
|
||||
" of iterations of loop %d; likely exits remains.\n",
|
||||
loop->num);
|
||||
|
||||
changed = true;
|
||||
}
|
||||
|
||||
|
||||
+198
-37
@@ -980,6 +980,75 @@ estimated_peeled_sequence_size (struct loop_size *size,
|
||||
- size->eliminated_by_peeling), 1);
|
||||
}
|
||||
|
||||
/* Update loop estimates after peeling LOOP by NPEEL.
|
||||
If PRECISE is false only likely exists were duplicated and thus
|
||||
do not update any estimates that are supposed to be always reliable. */
|
||||
void
|
||||
adjust_loop_info_after_peeling (class loop *loop, int npeel, bool precise)
|
||||
{
|
||||
if (loop->any_estimate)
|
||||
{
|
||||
/* Since peeling is mostly about loops where first few
|
||||
iterations are special, it is not quite correct to
|
||||
assume that the remaining iterations will behave
|
||||
the same way. However we do not have better info
|
||||
so update the esitmate, since it is likely better
|
||||
than keeping it as it is.
|
||||
|
||||
Remove it if it looks wrong.
|
||||
|
||||
TODO: We likely want to special case the situation where
|
||||
peeling is optimizing out exit edges and only update
|
||||
estimates here. */
|
||||
if (wi::leu_p (npeel, loop->nb_iterations_estimate))
|
||||
loop->nb_iterations_estimate -= npeel;
|
||||
else
|
||||
loop->any_estimate = false;
|
||||
}
|
||||
if (loop->any_upper_bound && precise)
|
||||
{
|
||||
if (wi::leu_p (npeel, loop->nb_iterations_upper_bound))
|
||||
loop->nb_iterations_upper_bound -= npeel;
|
||||
else
|
||||
{
|
||||
/* Peeling maximal number of iterations or more
|
||||
makes no sense and is a bug.
|
||||
We should peel completely. */
|
||||
gcc_unreachable ();
|
||||
}
|
||||
}
|
||||
if (loop->any_likely_upper_bound)
|
||||
{
|
||||
if (wi::leu_p (npeel, loop->nb_iterations_likely_upper_bound))
|
||||
loop->nb_iterations_likely_upper_bound -= npeel;
|
||||
else
|
||||
{
|
||||
loop->any_estimate = true;
|
||||
loop->nb_iterations_estimate = 0;
|
||||
loop->nb_iterations_likely_upper_bound = 0;
|
||||
}
|
||||
}
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_minus_upper_bound (loop->counters, npeel);
|
||||
/* TODO: Update loop estimate if possible. */
|
||||
}
|
||||
}
|
||||
|
||||
/* Return, in percent, probability that loop will end in first N iterations. */
|
||||
|
||||
static int
|
||||
loop_histogram_percentage (class loop *loop, int n)
|
||||
{
|
||||
gcov_type sum = loop->counters->sum;
|
||||
gcov_type psum = 0;
|
||||
if (n > (int)loop->counters->lin->length ())
|
||||
return -1;
|
||||
for (int i = 0; i < n; i++)
|
||||
psum += (*loop->counters->lin)[i];
|
||||
return RDIV (psum * 100, sum);
|
||||
}
|
||||
|
||||
/* If the loop is expected to iterate N times and is
|
||||
small enough, duplicate the loop body N+1 times before
|
||||
the loop itself. This way the hot path will never
|
||||
@@ -989,7 +1058,7 @@ estimated_peeled_sequence_size (struct loop_size *size,
|
||||
static bool
|
||||
try_peel_loop (class loop *loop,
|
||||
edge exit, tree niter, bool may_be_zero,
|
||||
HOST_WIDE_INT maxiter)
|
||||
HOST_WIDE_INT maxiter, dump_user_location_t locus)
|
||||
{
|
||||
HOST_WIDE_INT npeel;
|
||||
struct loop_size size;
|
||||
@@ -1037,6 +1106,78 @@ try_peel_loop (class loop *loop,
|
||||
npeel = estimated_loop_iterations_int (loop);
|
||||
if (npeel < 0)
|
||||
npeel = likely_max_loop_iterations_int (loop);
|
||||
auto_vec<int> good_peels;
|
||||
auto_vec<int> prcnt;
|
||||
prcnt.safe_push (0);
|
||||
bool histogram_peeling = flag_use_histogram_in_peeling && (loop->counters != NULL);
|
||||
if (!loop->counters && loop->header->count.reliable_p ()
|
||||
&& loop->header->count.nonzero_p ()
|
||||
&& dump_enabled_p ())
|
||||
dump_printf_loc (MSG_MISSED_OPTIMIZATION | MSG_PRIORITY_USER_FACING, locus,
|
||||
"missing loop histogram for loop %d.\n", loop->num);
|
||||
HOST_WIDE_INT nonhistogram_npeel = npeel + 1;
|
||||
if (histogram_peeling && loop->counters->sum != 0
|
||||
&& loop->counters->lin->length ())
|
||||
{
|
||||
gcov_type sum = loop->counters->sum;
|
||||
gcov_type rest = sum;
|
||||
gcov_type psum = 0;
|
||||
gcov_type osum = 0;
|
||||
int good_percentage = param_profile_histogram_peel_prcnt;
|
||||
int good_overall_percentage = param_profile_histogram_peel_overall_prcnt;
|
||||
for (int i = 0; i < (int)loop->counters->lin->length (); i++)
|
||||
{
|
||||
gcov_type e = (*(loop->counters->lin))[i];
|
||||
psum += e;
|
||||
osum += e;
|
||||
if ((maxiter >= 0 && maxiter <= i)
|
||||
|| (i >= param_max_peel_times - 1)
|
||||
|| rest == 0)
|
||||
break;
|
||||
// iteration has enough cumulated in partial sum and itself has at
|
||||
// least 1 percent or we have complete peeling
|
||||
if (e && ((100 * psum) / sum >= good_percentage || psum == rest)
|
||||
&& ((100 * osum) / sum) >= good_overall_percentage)
|
||||
{
|
||||
prcnt.safe_push (prcnt.last () + (100 * psum) / sum);
|
||||
good_peels.safe_push (i);
|
||||
good_percentage = 0;
|
||||
rest -= psum;
|
||||
psum = 0;
|
||||
}
|
||||
good_percentage = MIN (100, good_percentage + param_profile_histogram_peel_prcnt);
|
||||
}
|
||||
if (!good_peels.is_empty ())
|
||||
{
|
||||
npeel = good_peels.pop ();
|
||||
// we do not pop() prcnt because we want it to be last for current
|
||||
// iteration
|
||||
}
|
||||
else
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file,
|
||||
"Not peeling: found no good candidates in the histogram\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!flag_peel_loops_without_histogram)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Not peeling: histogram is not available\n");
|
||||
return false;
|
||||
}
|
||||
if (npeel >= 0 && maxiter >= 0 && maxiter <= npeel)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Not peeling: upper bound is known so can "
|
||||
"unroll completely\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (npeel < 0)
|
||||
{
|
||||
if (dump_file)
|
||||
@@ -1044,17 +1185,11 @@ try_peel_loop (class loop *loop,
|
||||
"estimated\n");
|
||||
return false;
|
||||
}
|
||||
if (maxiter >= 0 && maxiter <= npeel)
|
||||
{
|
||||
if (dump_file)
|
||||
fprintf (dump_file, "Not peeling: upper bound is known so can "
|
||||
"unroll completely\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
/* We want to peel estimated number of iterations + 1 (so we never
|
||||
enter the loop on quick path). Check against PARAM_MAX_PEEL_TIMES
|
||||
and be sure to avoid overflows. */
|
||||
|
||||
if (npeel > param_max_peel_times - 1)
|
||||
{
|
||||
if (dump_file)
|
||||
@@ -1067,6 +1202,19 @@ try_peel_loop (class loop *loop,
|
||||
/* Check peeled loops size. */
|
||||
tree_estimate_loop_size (loop, exit, NULL, &size,
|
||||
param_max_peeled_insns);
|
||||
// we want to peel the max_peeled_insns
|
||||
// if we peel percentage corresponding the whole linear section
|
||||
while (!good_peels.is_empty ()
|
||||
&& ((int) estimated_peeled_sequence_size (&size, (int) npeel)
|
||||
* MIN (100, param_profile_histogram_size_lin
|
||||
* param_profile_histogram_peel_prcnt)
|
||||
> param_max_peeled_insns * (prcnt.last ())))
|
||||
{
|
||||
prcnt.pop ();
|
||||
npeel = good_peels.pop ();
|
||||
++npeel;
|
||||
}
|
||||
|
||||
if ((peeled_size = estimated_peeled_sequence_size (&size, (int) npeel))
|
||||
> param_max_peeled_insns)
|
||||
{
|
||||
@@ -1104,36 +1252,49 @@ try_peel_loop (class loop *loop,
|
||||
return false;
|
||||
}
|
||||
free_original_copy_tables ();
|
||||
if (dump_file && (dump_flags & TDF_DETAILS))
|
||||
|
||||
if (dump_enabled_p ())
|
||||
{
|
||||
fprintf (dump_file, "Peeled loop %d, %i times.\n",
|
||||
loop->num, (int) npeel);
|
||||
}
|
||||
if (loop->any_estimate)
|
||||
{
|
||||
if (wi::ltu_p (npeel, loop->nb_iterations_estimate))
|
||||
loop->nb_iterations_estimate -= npeel;
|
||||
else
|
||||
loop->nb_iterations_estimate = 0;
|
||||
}
|
||||
if (loop->any_upper_bound)
|
||||
{
|
||||
if (wi::ltu_p (npeel, loop->nb_iterations_upper_bound))
|
||||
loop->nb_iterations_upper_bound -= npeel;
|
||||
else
|
||||
loop->nb_iterations_upper_bound = 0;
|
||||
}
|
||||
if (loop->any_likely_upper_bound)
|
||||
{
|
||||
if (wi::ltu_p (npeel, loop->nb_iterations_likely_upper_bound))
|
||||
loop->nb_iterations_likely_upper_bound -= npeel;
|
||||
else
|
||||
bool close = false;
|
||||
if (histogram_peeling)
|
||||
{
|
||||
loop->any_estimate = true;
|
||||
loop->nb_iterations_estimate = 0;
|
||||
loop->nb_iterations_likely_upper_bound = 0;
|
||||
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, locus,
|
||||
"peeled loop %d, %i times with histogram, %i%% of executions",
|
||||
loop->num, (int)npeel, loop_histogram_percentage (loop, npeel));
|
||||
if (nonhistogram_npeel != npeel)
|
||||
{
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
|
||||
" (without histogram would try to peel %i times",
|
||||
(int)nonhistogram_npeel);
|
||||
int p = loop_histogram_percentage (loop, nonhistogram_npeel);
|
||||
if (p >= 0)
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
|
||||
", %i%% of executions", p);
|
||||
close = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, locus,
|
||||
"peeled loop %d, %i times", loop->num, (int)npeel);
|
||||
if (profile_info && loop->header->count.ipa ().initialized_p ())
|
||||
{
|
||||
if (!close)
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, " (");
|
||||
else
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, "; ");
|
||||
close = true;
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
|
||||
"header execution count %wd",
|
||||
(HOST_WIDE_INT)loop->header->count.ipa ().to_gcov_type ());
|
||||
}
|
||||
if (close)
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, ")\n");
|
||||
else
|
||||
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, "\n");
|
||||
}
|
||||
|
||||
adjust_loop_info_after_peeling (loop, npeel, true);
|
||||
|
||||
profile_count entry_count = profile_count::zero ();
|
||||
|
||||
edge e;
|
||||
@@ -1151,6 +1312,7 @@ try_peel_loop (class loop *loop,
|
||||
bitmap_set_bit (peeled_loops, loop->num);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Adds a canonical induction variable to LOOP if suitable.
|
||||
CREATE_IV is true if we may create a new iv. UL determines
|
||||
which loops we are allowed to completely unroll. If TRY_EVAL is true, we try
|
||||
@@ -1273,9 +1435,8 @@ canonicalize_loop_induction_variables (class loop *loop,
|
||||
if (iv_niter)
|
||||
create_canonical_iv (loop, exit, iv_niter);
|
||||
}
|
||||
|
||||
if (ul == UL_ALL)
|
||||
modified |= try_peel_loop (loop, exit, niter, may_be_zero, maxiter);
|
||||
modified |= try_peel_loop (loop, exit, niter, may_be_zero, maxiter, locus);
|
||||
|
||||
return modified;
|
||||
}
|
||||
@@ -1600,7 +1761,7 @@ pass_complete_unroll::execute (function *fun)
|
||||
re-peeling the same loop multiple times. */
|
||||
if (flag_peel_loops)
|
||||
peeled_loops = BITMAP_ALLOC (NULL);
|
||||
unsigned int val = tree_unroll_loops_completely (flag_cunroll_grow_size,
|
||||
unsigned int val = tree_unroll_loops_completely (flag_cunroll_grow_size,
|
||||
true);
|
||||
if (peeled_loops)
|
||||
{
|
||||
|
||||
@@ -67,7 +67,9 @@ create_iv (tree base, tree step, tree var, class loop *loop,
|
||||
gimple_seq stmts;
|
||||
tree vb, va;
|
||||
enum tree_code incr_op = PLUS_EXPR;
|
||||
edge pe = loop_preheader_edge (loop);
|
||||
edge pe = NULL;
|
||||
edge e;
|
||||
edge_iterator ei;
|
||||
|
||||
if (var != NULL_TREE)
|
||||
{
|
||||
@@ -122,7 +124,11 @@ create_iv (tree base, tree step, tree var, class loop *loop,
|
||||
loop (i.e. the step should be loop invariant). */
|
||||
step = force_gimple_operand (step, &stmts, true, NULL_TREE);
|
||||
if (stmts)
|
||||
gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
{
|
||||
if (!pe)
|
||||
pe = loop_preheader_edge (loop);
|
||||
gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
}
|
||||
|
||||
stmt = gimple_build_assign (va, incr_op, vb, step);
|
||||
/* Prevent the increment from inheriting a bogus location if it is not put
|
||||
@@ -151,11 +157,24 @@ create_iv (tree base, tree step, tree var, class loop *loop,
|
||||
|
||||
initial = force_gimple_operand (base, &stmts, true, var);
|
||||
if (stmts)
|
||||
gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
{
|
||||
if (!pe)
|
||||
pe = loop_preheader_edge (loop);
|
||||
gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
}
|
||||
|
||||
phi = create_phi_node (vb, loop->header);
|
||||
add_phi_arg (phi, initial, loop_preheader_edge (loop), UNKNOWN_LOCATION);
|
||||
add_phi_arg (phi, va, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
||||
FOR_EACH_EDGE (e, ei, loop->header->preds)
|
||||
{
|
||||
if (e == loop_latch_edge (loop))
|
||||
{
|
||||
add_phi_arg (phi, va, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
||||
}
|
||||
else
|
||||
{
|
||||
add_phi_arg (phi, initial, e, UNKNOWN_LOCATION);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Return the innermost superloop LOOP of USE_LOOP that is a superloop of
|
||||
|
||||
@@ -2941,6 +2941,17 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
|
||||
/* Prolog iterates at most bound_prolog times, latch iterates at
|
||||
most bound_prolog - 1 times. */
|
||||
record_niter_bound (prolog, bound_prolog - 1, false, true);
|
||||
/* TODO: Update counters or at least release vectors.
|
||||
We do not produce prologs on modern x86 targets we test on
|
||||
so this is not important for proof of concept. */
|
||||
if (prolog->counters)
|
||||
{
|
||||
va_heap::release (prolog->counters->lin);
|
||||
va_heap::release (prolog->counters->exp);
|
||||
if (prolog->counters->mod)
|
||||
va_heap::release (prolog->counters->mod);
|
||||
prolog->counters = NULL;
|
||||
}
|
||||
delete_update_ssa ();
|
||||
adjust_vec_debug_stmts ();
|
||||
scev_reset ();
|
||||
@@ -3077,6 +3088,15 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
|
||||
/* -1 to convert loop iterations to latch iterations. */
|
||||
record_niter_bound (epilog, bound - 1, false, true);
|
||||
}
|
||||
/* TODO: Compute epilog histograms or at least release the counters. */
|
||||
if (epilog->counters)
|
||||
{
|
||||
va_heap::release (epilog->counters->lin);
|
||||
va_heap::release (epilog->counters->exp);
|
||||
if (epilog->counters->mod)
|
||||
va_heap::release (epilog->counters->mod);
|
||||
epilog->counters = NULL;
|
||||
}
|
||||
|
||||
delete_update_ssa ();
|
||||
adjust_vec_debug_stmts ();
|
||||
|
||||
+48
-1
@@ -2070,6 +2070,38 @@ vect_analyze_loop_costing (loop_vec_info loop_vinfo,
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool use_estimate = true;
|
||||
/* With histogram we can determine how often the vectorized loop will be
|
||||
reached. */
|
||||
if (loop->counters
|
||||
/* TODO: For epilogue we can compute new histogram. */
|
||||
&& !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
|
||||
&& loop->counters->sum
|
||||
&& min_profitable_estimate <= (int)loop->counters->lin->length ())
|
||||
{
|
||||
gcov_type psum = 0;
|
||||
for (int i = 0; i < min_profitable_estimate
|
||||
&& i < (int) loop->counters->lin->length ();
|
||||
i++)
|
||||
psum += (*(loop->counters->lin))[i];
|
||||
/* TODO: This is less precise than needed. If histogram was also
|
||||
storing number of executions of the loop header (which may
|
||||
be different from loop->header->count if i.e. inlining happen).
|
||||
We could compute expected nunber of execution of the vectorized
|
||||
loop. */
|
||||
if (psum == loop->counters->sum
|
||||
|| !maybe_hot_count_p (cfun, loop->header->count.apply_scale
|
||||
(loop->counters->sum - psum,
|
||||
loop->counters->sum)))
|
||||
{
|
||||
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
|
||||
"not vectorized: according to loop histogram "
|
||||
"the vectorized loop would be cold.\n");
|
||||
return 0;
|
||||
}
|
||||
use_estimate = false;
|
||||
}
|
||||
|
||||
HOST_WIDE_INT estimated_niter;
|
||||
|
||||
/* If we are vectorizing an epilogue then we know the maximum number of
|
||||
@@ -2080,7 +2112,8 @@ vect_analyze_loop_costing (loop_vec_info loop_vinfo,
|
||||
= vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) - 1;
|
||||
else
|
||||
{
|
||||
estimated_niter = estimated_stmt_executions_int (loop);
|
||||
if (use_estimate)
|
||||
estimated_niter = estimated_stmt_executions_int (loop);
|
||||
if (estimated_niter == -1)
|
||||
estimated_niter = likely_max_stmt_executions_int (loop);
|
||||
}
|
||||
@@ -11056,6 +11089,11 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
|
||||
bias_for_lowest += lowest_vf - min_first_active;
|
||||
bias_for_assumed += assumed_vf - min_first_active;
|
||||
}
|
||||
if (loop->counters)
|
||||
{
|
||||
histogram_counters_div_upper_bound (loop->counters, lowest_vf);
|
||||
histogram_counters_minus_upper_bound (loop->counters, 1);
|
||||
}
|
||||
/* In these calculations the "- 1" converts loop iteration counts
|
||||
back to latch counts. */
|
||||
if (loop->any_upper_bound)
|
||||
@@ -11102,6 +11140,15 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
|
||||
assumed_vf) - 1
|
||||
: wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
|
||||
assumed_vf) - 1);
|
||||
/* TODO: Fix profile after vectorization. */
|
||||
if (loop->counters)
|
||||
{
|
||||
va_heap::release (loop->counters->lin);
|
||||
va_heap::release (loop->counters->exp);
|
||||
if (loop->counters->mod)
|
||||
va_heap::release (loop->counters->mod);
|
||||
loop->counters = NULL;
|
||||
}
|
||||
|
||||
if (dump_enabled_p ())
|
||||
{
|
||||
|
||||
+124
-6
@@ -42,6 +42,10 @@ along with GCC; see the file COPYING3. If not see
|
||||
#include "gimple-pretty-print.h"
|
||||
#include "dumpfile.h"
|
||||
#include "builtins.h"
|
||||
#include "cfgloop.h"
|
||||
#include "tree-ssa-loop-manip.h"
|
||||
#include "tree-ssa-loop.h"
|
||||
#include "tree-ssa-loop-niter.h"
|
||||
|
||||
/* In this file value profile based optimizations are placed. Currently the
|
||||
following optimizations are implemented (for more detailed descriptions
|
||||
@@ -117,6 +121,20 @@ gimple_alloc_histogram_value (struct function *fun ATTRIBUTE_UNUSED,
|
||||
histogram_value hist = (histogram_value) xcalloc (1, sizeof (*hist));
|
||||
hist->hvalue.value = value;
|
||||
hist->hvalue.stmt = stmt;
|
||||
hist->hvalue.lp = NULL;
|
||||
hist->type = type;
|
||||
return hist;
|
||||
}
|
||||
|
||||
histogram_value
|
||||
gimple_alloc_histogram_value_loop (struct function *fun ATTRIBUTE_UNUSED,
|
||||
enum hist_type type, tree value,
|
||||
class loop *lp)
|
||||
{
|
||||
histogram_value hist = (histogram_value) xcalloc (1, sizeof (*hist));
|
||||
hist->hvalue.value = value;
|
||||
hist->hvalue.stmt = NULL;
|
||||
hist->hvalue.lp = lp;
|
||||
hist->type = type;
|
||||
return hist;
|
||||
}
|
||||
@@ -247,6 +265,38 @@ dump_histogram_value (FILE *dump_file, histogram_value hist)
|
||||
(int64_t) hist->hvalue.counters[i]);
|
||||
}
|
||||
break;
|
||||
case HIST_TYPE_HISTOGRAM:
|
||||
if (hist->hvalue.counters)
|
||||
{
|
||||
for (int i = 0; i <= param_profile_histogram_size_lin; ++i)
|
||||
{
|
||||
fprintf (dump_file,
|
||||
"Histogram counter histogram%" PRId64 ":%" PRId64 ".\n",
|
||||
(int64_t) i, (int64_t) hist->hvalue.counters[i]);
|
||||
}
|
||||
int lin2_log = floor_log2 (param_profile_histogram_size_lin);
|
||||
for (int64_t i = lin2_log;
|
||||
i < param_profile_histogram_size_exp + lin2_log; ++i)
|
||||
{
|
||||
fprintf (
|
||||
dump_file,
|
||||
"Histogram counter histogram%" PRId64 ":%" PRId64 ".\n",
|
||||
(int64_t) 1 << i,
|
||||
(int64_t) hist->hvalue
|
||||
.counters[(param_profile_histogram_size_lin - lin2_log) + i]);
|
||||
}
|
||||
for (int i = 0; i < param_profile_histogram_size_mod; ++i)
|
||||
{
|
||||
fprintf (dump_file,
|
||||
"Histogram counter histogram%" PRId64 " mod:%" PRId64
|
||||
".\n",
|
||||
(int64_t) i,
|
||||
(int64_t) hist->hvalue
|
||||
.counters[i + param_profile_histogram_size_lin
|
||||
+ param_profile_histogram_size_exp]);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case HIST_TYPE_POW2:
|
||||
if (hist->hvalue.counters)
|
||||
@@ -364,6 +414,11 @@ stream_in_histogram_value (class lto_input_block *ib, gimple *stmt)
|
||||
ncounters = new_val->hdata.intvl.steps + 2;
|
||||
break;
|
||||
|
||||
case HIST_TYPE_HISTOGRAM:
|
||||
ncounters = param_profile_histogram_size_lin
|
||||
+ param_profile_histogram_size_exp
|
||||
+ param_profile_histogram_size_mod;
|
||||
break;
|
||||
case HIST_TYPE_POW2:
|
||||
case HIST_TYPE_AVERAGE:
|
||||
ncounters = 2;
|
||||
@@ -1799,11 +1854,13 @@ gimple_divmod_values_to_profile (gimple *stmt, histogram_values *values)
|
||||
op0 = gimple_assign_rhs1 (stmt);
|
||||
|
||||
if (TREE_CODE (divisor) == SSA_NAME)
|
||||
/* Check for the case where the divisor is the same value most
|
||||
of the time. */
|
||||
values->safe_push (gimple_alloc_histogram_value (cfun,
|
||||
HIST_TYPE_TOPN_VALUES,
|
||||
stmt, divisor));
|
||||
{
|
||||
/* Check for the case where the divisor is the same value most
|
||||
of the time. */
|
||||
values->safe_push (
|
||||
gimple_alloc_histogram_value (cfun, HIST_TYPE_TOPN_VALUES, stmt,
|
||||
divisor));
|
||||
}
|
||||
|
||||
/* For mod, check whether it is not often a noop (or replaceable by
|
||||
a few subtractions). */
|
||||
@@ -1816,6 +1873,7 @@ gimple_divmod_values_to_profile (gimple *stmt, histogram_values *values)
|
||||
values->safe_push (gimple_alloc_histogram_value (cfun,
|
||||
HIST_TYPE_POW2,
|
||||
stmt, divisor));
|
||||
|
||||
val = build2 (TRUNC_DIV_EXPR, type, op0, divisor);
|
||||
hist = gimple_alloc_histogram_value (cfun, HIST_TYPE_INTERVAL,
|
||||
stmt, val);
|
||||
@@ -1889,6 +1947,60 @@ gimple_stringops_values_to_profile (gimple *gs, histogram_values *values)
|
||||
stmt, dest));
|
||||
}
|
||||
|
||||
static void
|
||||
gimple_histogram_values_to_profile (function *fun, histogram_values *values)
|
||||
{
|
||||
if (!flag_profile_loops)
|
||||
return;
|
||||
for (auto loop : loops_list (fun, 0))
|
||||
{
|
||||
tree var;
|
||||
gimple_stmt_iterator gsi;
|
||||
edge exit_edge = NULL;
|
||||
auto_vec<edge> exits = get_loop_exit_edges (loop);
|
||||
bool complex_exit = false;
|
||||
bool exit_found = false;
|
||||
|
||||
/* If loop has only exits that can not be instrumented,
|
||||
we can not profile histograms. */
|
||||
for (auto exit : exits)
|
||||
if (exit->flags & EDGE_FAKE)
|
||||
;
|
||||
else if (exit->flags & EDGE_COMPLEX)
|
||||
complex_exit = true;
|
||||
else
|
||||
{
|
||||
if (!exit_found)
|
||||
exit_edge = exit;
|
||||
else
|
||||
{
|
||||
exit_edge = NULL;
|
||||
break;
|
||||
}
|
||||
exit_found = true;
|
||||
}
|
||||
if (!exit_found)
|
||||
continue;
|
||||
|
||||
/* If there is only one non-fake exit edge, see if we know number of
|
||||
iterations. */
|
||||
tree niter;
|
||||
edge ex;
|
||||
if (!complex_exit && exit_edge
|
||||
&& (niter = find_loop_niter (loop, &ex))
|
||||
&& TREE_CODE (niter) == INTEGER_CST)
|
||||
continue;
|
||||
|
||||
gsi = gsi_start_bb (loop->latch);
|
||||
create_iv (build_int_cst_type (get_gcov_type (), 0),
|
||||
build_int_cst (get_gcov_type (), 1), NULL_TREE, loop, &gsi,
|
||||
true, &var, NULL);
|
||||
values->safe_push (gimple_alloc_histogram_value_loop (fun,
|
||||
HIST_TYPE_HISTOGRAM,
|
||||
var, loop));
|
||||
}
|
||||
}
|
||||
|
||||
/* Find values inside STMT for that we want to measure histograms and adds
|
||||
them to list VALUES. */
|
||||
|
||||
@@ -1908,7 +2020,7 @@ gimple_find_values_to_profile (histogram_values *values)
|
||||
unsigned i;
|
||||
histogram_value hist = NULL;
|
||||
values->create (0);
|
||||
|
||||
gimple_histogram_values_to_profile (cfun, values);
|
||||
FOR_EACH_BB_FN (bb, cfun)
|
||||
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
||||
gimple_values_to_profile (gsi_stmt (gsi), values);
|
||||
@@ -1945,6 +2057,12 @@ gimple_find_values_to_profile (histogram_values *values)
|
||||
hist->n_counters = 1;
|
||||
break;
|
||||
|
||||
case HIST_TYPE_HISTOGRAM:
|
||||
hist->n_counters = param_profile_histogram_size_lin
|
||||
+ param_profile_histogram_size_exp
|
||||
+ param_profile_histogram_size_mod;
|
||||
break;
|
||||
|
||||
default:
|
||||
gcc_unreachable ();
|
||||
}
|
||||
|
||||
+16
-11
@@ -23,15 +23,16 @@ along with GCC; see the file COPYING3. If not see
|
||||
/* Supported histogram types. */
|
||||
enum hist_type
|
||||
{
|
||||
HIST_TYPE_INTERVAL, /* Measures histogram of values inside a specified
|
||||
interval. */
|
||||
HIST_TYPE_POW2, /* Histogram of power of 2 values. */
|
||||
HIST_TYPE_TOPN_VALUES, /* Tries to identify the N most common values. */
|
||||
HIST_TYPE_INDIR_CALL, /* Tries to identify the function that is (almost)
|
||||
HIST_TYPE_INTERVAL, /* Measures histogram of values inside a specified
|
||||
interval. */
|
||||
HIST_TYPE_POW2, /* Histogram of power of 2 values. */
|
||||
HIST_TYPE_TOPN_VALUES, /* Tries to identify the N most common values. */
|
||||
HIST_TYPE_INDIR_CALL, /* Tries to identify the function that is (almost)
|
||||
called in indirect call */
|
||||
HIST_TYPE_AVERAGE, /* Compute average value (sum of all values). */
|
||||
HIST_TYPE_IOR, /* Used to compute expected alignment. */
|
||||
HIST_TYPE_AVERAGE, /* Compute average value (sum of all values). */
|
||||
HIST_TYPE_IOR, /* Used to compute expected alignment. */
|
||||
HIST_TYPE_TIME_PROFILE, /* Used for time profile */
|
||||
HIST_TYPE_HISTOGRAM, /* used values histogram */
|
||||
HIST_TYPE_MAX
|
||||
};
|
||||
|
||||
@@ -39,7 +40,8 @@ enum hist_type
|
||||
#define HIST_TYPE_FOR_COUNTER(COUNTER) \
|
||||
((enum hist_type) ((COUNTER) - GCOV_FIRST_VALUE_COUNTER))
|
||||
|
||||
|
||||
class loop;
|
||||
|
||||
/* The value to measure. */
|
||||
struct histogram_value_t
|
||||
{
|
||||
@@ -47,6 +49,7 @@ struct histogram_value_t
|
||||
{
|
||||
tree value; /* The value to profile. */
|
||||
gimple *stmt; /* Insn containing the value. */
|
||||
class loop *lp;
|
||||
gcov_type *counters; /* Pointer to first counter. */
|
||||
struct histogram_value_t *next; /* Linked list pointer. */
|
||||
} hvalue;
|
||||
@@ -72,9 +75,9 @@ typedef vec<histogram_value> histogram_values;
|
||||
extern void gimple_find_values_to_profile (histogram_values *);
|
||||
extern bool gimple_value_profile_transformations (void);
|
||||
|
||||
histogram_value gimple_alloc_histogram_value (struct function *, enum hist_type,
|
||||
gimple *stmt = NULL,
|
||||
tree value = NULL_TREE);
|
||||
histogram_value
|
||||
gimple_alloc_histogram_value (struct function *, enum hist_type,
|
||||
gimple *stmt = NULL, tree value = NULL);
|
||||
histogram_value gimple_histogram_value (struct function *, gimple *);
|
||||
histogram_value gimple_histogram_value_of_type (struct function *, gimple *,
|
||||
enum hist_type);
|
||||
@@ -99,6 +102,8 @@ extern void gimple_init_gcov_profiler (void);
|
||||
extern void gimple_gen_edge_profiler (int, edge);
|
||||
extern void gimple_gen_interval_profiler (histogram_value, unsigned);
|
||||
extern void gimple_gen_pow2_profiler (histogram_value, unsigned);
|
||||
extern void
|
||||
gimple_gen_histogram_profiler (histogram_value, unsigned);
|
||||
extern void gimple_gen_topn_values_profiler (histogram_value, unsigned);
|
||||
extern void gimple_gen_ic_profiler (histogram_value, unsigned);
|
||||
extern void gimple_gen_ic_func_profiler (void);
|
||||
|
||||
+3
-1
@@ -906,7 +906,9 @@ LIBGCOV_PROFILER = _gcov_interval_profiler \
|
||||
_gcov_ior_profiler \
|
||||
_gcov_ior_profiler_atomic \
|
||||
_gcov_indirect_call_profiler_v4 \
|
||||
_gcov_time_profiler
|
||||
_gcov_time_profiler \
|
||||
_gcov_histogram_profiler \
|
||||
_gcov_histogram_profiler_atomic
|
||||
LIBGCOV_INTERFACE = _gcov_dump _gcov_fork \
|
||||
_gcov_execl _gcov_execlp \
|
||||
_gcov_execle _gcov_execv _gcov_execvp _gcov_execve _gcov_reset \
|
||||
|
||||
@@ -26,6 +26,103 @@ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
||||
#include "libgcov.h"
|
||||
#if !defined(inhibit_libc)
|
||||
|
||||
#ifndef floor_log2
|
||||
|
||||
/*
|
||||
* If value is less then 8 we increment corresponding counter
|
||||
* otherwise we take its logarithm and increment corresponding counter
|
||||
*/
|
||||
|
||||
/* For convenience, define 0 -> word_size. */
|
||||
static inline int clz_hwi(gcov_type_unsigned x) {
|
||||
if (x == 0)
|
||||
return sizeof(gcov_type_unsigned) * 8;
|
||||
if (sizeof(gcov_type_unsigned) == sizeof(long))
|
||||
return __builtin_clzl(x);
|
||||
if (sizeof(gcov_type_unsigned) == sizeof(long long))
|
||||
return __builtin_clzll(x);
|
||||
else
|
||||
return __builtin_clz(x);
|
||||
}
|
||||
|
||||
static inline gcov_type_unsigned floor_log2(gcov_type_unsigned x) {
|
||||
return sizeof(gcov_type_unsigned) * 8 - clz_hwi(x) - 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef L_gcov_histogram_profiler
|
||||
/*
|
||||
* If value is less then 8 we increment corresponding counter
|
||||
* otherwise we take its logarithm and increment corresponding counter
|
||||
*/
|
||||
|
||||
void __gcov_histogram_profiler(gcov_type *counters, gcov_type value,
|
||||
gcov_type hist_sizes) {
|
||||
gcc_assert(hist_sizes >= 0 && value >= 0);
|
||||
gcov_type_unsigned hist_size = hist_sizes;
|
||||
gcov_type_unsigned u_value = value;
|
||||
// uncode section sizes
|
||||
gcov_type_unsigned mask = (((gcov_type_unsigned)1) << 10) - 1;
|
||||
gcov_type_unsigned lin_size = hist_size & mask;
|
||||
gcov_type_unsigned exp_size = (hist_size >> 10) & mask;
|
||||
gcov_type_unsigned mod_size = hist_size >> 20;
|
||||
gcov_type_unsigned tot_size = exp_size + lin_size;
|
||||
// add to the regular histogram
|
||||
if (u_value < lin_size) {
|
||||
counters[value]++;
|
||||
} else {
|
||||
gcov_type_unsigned pow2 = floor_log2(u_value);
|
||||
gcov_type_unsigned lin_pow2 = floor_log2(lin_size - 1);
|
||||
if (lin_size < tot_size && pow2 == lin_pow2) {
|
||||
counters[lin_size]++;
|
||||
} else {
|
||||
if ((lin_pow2 - lin_size) + tot_size > pow2) {
|
||||
counters[pow2 + (lin_size - lin_pow2) - 1]++;
|
||||
} else {
|
||||
counters[tot_size - 1]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// add to the modular histogram
|
||||
counters[tot_size + (u_value % mod_size)]++;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(L_gcov_histogram_profiler_atomic) && GCOV_SUPPORTS_ATOMIC
|
||||
|
||||
void __gcov_histogram_profiler_atomic(gcov_type *counters, gcov_type value,
|
||||
gcov_type hist_sizes) {
|
||||
gcc_assert(hist_sizes >= 0 && value >= 0);
|
||||
gcov_type_unsigned hist_size = hist_sizes;
|
||||
gcov_type_unsigned u_value = value;
|
||||
// uncode section sizes
|
||||
gcov_type_unsigned mask = (((gcov_type_unsigned)1) << 10) - 1;
|
||||
gcov_type_unsigned lin_size = hist_size & mask;
|
||||
gcov_type_unsigned exp_size = (hist_size >> 10) & mask;
|
||||
gcov_type_unsigned mod_size = hist_size >> 20;
|
||||
gcov_type_unsigned tot_size = exp_size + lin_size;
|
||||
// add to the regular histogram
|
||||
if (u_value < lin_size) {
|
||||
__atomic_fetch_add(&counters[value], 1, __ATOMIC_RELAXED);
|
||||
} else {
|
||||
gcov_type_unsigned pow2 = floor_log2(u_value);
|
||||
gcov_type_unsigned lin_pow2 = floor_log2(lin_size - 1);
|
||||
if ((lin_pow2 - lin_size) + tot_size > pow2) {
|
||||
__atomic_fetch_add(&counters[pow2 + (lin_size - lin_pow2) - 1], 1,
|
||||
__ATOMIC_RELAXED);
|
||||
} else {
|
||||
__atomic_fetch_add(&counters[tot_size - 1], 1, __ATOMIC_RELAXED);
|
||||
}
|
||||
}
|
||||
// add to the modular histogram
|
||||
__atomic_fetch_add(&counters[tot_size + (u_value % mod_size)], 1,
|
||||
__ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef L_gcov_interval_profiler
|
||||
/* If VALUE is in interval <START, START + STEPS - 1>, then increases the
|
||||
corresponding counter in COUNTERS. If the VALUE is above or below
|
||||
|
||||
@@ -317,6 +317,8 @@ extern void __gcov_merge_ior (gcov_type *, unsigned) ATTRIBUTE_HIDDEN;
|
||||
extern void __gcov_interval_profiler (gcov_type *, gcov_type, int, unsigned);
|
||||
extern void __gcov_interval_profiler_atomic (gcov_type *, gcov_type, int,
|
||||
unsigned);
|
||||
extern void __gcov_histogram_profiler(gcov_type *, gcov_type, gcov_type);
|
||||
extern void __gcov_histogram_profiler_atomic(gcov_type *, gcov_type, gcov_type);
|
||||
extern void __gcov_pow2_profiler (gcov_type *, gcov_type);
|
||||
extern void __gcov_pow2_profiler_atomic (gcov_type *, gcov_type);
|
||||
extern void __gcov_topn_values_profiler (gcov_type *, gcov_type);
|
||||
|
||||
Reference in New Issue
Block a user