Compare commits

...
85 Commits
Author SHA1 Message Date
kubaneko 24715cc85f fixed typo 2023-05-09 20:10:59 +00:00
Honza 5cbb9db198 Add parameters to control histogram profiling and peeling 2023-05-09 16:26:00 +02:00
Honza 2ad0725272 Minor heuristics improvements 2023-05-07 21:47:26 +02:00
Honza 79ac8f606a Do not use params to determine histogram sizes 2023-05-07 21:45:42 +02:00
Honza 01bce22104 Do not use params for determining histogram sizes 2023-05-07 21:44:46 +02:00
Honza 9ce81bff85 Fix reading of histograms. 2023-05-07 21:44:07 +02:00
Honza 746af9947a Fix bug in identifying number of exits of the loop. 2023-05-07 21:42:59 +02:00
Honza d6bb96e3a1 Better -fopt-info 2023-05-07 17:58:54 +02:00
kubaneko cc39273df2 constrained the parameters 2023-05-02 20:03:19 +00:00
kubaneko 3884cb7b0b divided histogram_counters.hist into two .exp and .lin, changed histogram maintaining so exp does not try to recover lin and added some cleanup to histogram versioning 2023-05-02 18:24:16 +00:00
Honza 8667da401c Clear profile in vectorizer. 2023-05-02 00:39:34 +02:00
Honza e3bc373e2b Fix use of estimates. 2023-05-02 00:18:57 +02:00
Honza e4287fa88d Use histogram in loop vectorizer heuristics 2023-05-02 00:13:52 +02:00
Honza b37d3aa265 Do not instrument loops with known number of iterations and with no instrumentable exits. 2023-05-01 23:50:31 +02:00
Honza 349581e9d5 Add histogram based loop versioning. 2023-05-01 21:52:20 +02:00
kubaneko 3b7b26109c added modulo histogram and improved garbage collection 2023-04-25 08:09:35 +00:00
kubaneko 9a22679c15 changed from total size to exponential size 2023-04-24 07:54:30 +00:00
Honza 54eac33780 Add -fuse-histogram-in-peeling 2023-04-22 12:49:51 +02:00
Honza 3d73340a29 Merge final fix for copy header from master 2023-04-22 11:28:41 +02:00
Honza 9acda1e1fd Improve optimization info on peeling. 2023-04-22 11:05:01 +02:00
kubaneko 1525af68c6 hubicka bug fix 2023-04-21 15:34:54 +00:00
kubaneko 252ba6ce07 aggresive histogram 2023-04-20 14:48:35 +00:00
kubaneko 39612a0f22 division bugfix 2023-04-18 15:36:45 +00:00
kubaneko 9eb13a4658 bug fix 2023-04-15 20:32:57 +00:00
kubaneko 042619c2d7 gnu formating 2023-04-15 15:32:31 +00:00
kubaneko 0e92ac04b7 fixed warnings 2023-04-14 14:46:41 +00:00
kubaneko b4dd91d975 remove histograms when the sum is 0 2023-04-13 16:56:13 +00:00
kubaneko 692d26f8cc fixed some bugs 2023-04-13 15:31:32 +00:00
kubaneko c1d88b56c6 peeling bug, copy header, precision adjusting 2023-04-08 11:32:48 +00:00
kubaneko d05be13a4e memory management of peeling 2023-04-08 08:08:13 +00:00
kubaneko 085e3f70c9 fixed peeling bug 2023-04-07 21:45:09 +00:00
kubaneko e59f582bbb repaired copy header for histograms 2023-04-07 19:34:43 +00:00
kubaneko 03be06fc3f extracted adjust_loop_estimates_minus 2023-04-07 18:04:30 +00:00
kubaneko 801734b5a5 fixed a bug 2023-04-07 10:20:08 +00:00
kubaneko dd74f36d80 added adjusted flag 2023-04-07 08:26:52 +00:00
kubaneko 98bb27047d fixed bug 2023-04-05 15:50:26 +00:00
kubaneko 66ea24295b worst case adjusted 2023-04-05 15:42:12 +00:00
kubaneko cc896fddcc renaming and comments 2023-04-05 14:50:16 +00:00
kubaneko 94ab356577 fixed bugs 2023-04-05 14:44:52 +00:00
kubaneko 6f9a710ee8 fixed bug 2023-04-03 13:42:46 +00:00
kubaneko 6e7dfd5a75 change of peeled probabilities 2023-04-02 15:03:07 +00:00
kubaneko 0f991d297f peeling improvements 2023-03-29 13:40:21 +00:00
kubaneko bdcf0b0384 fixed histogram loading 2023-03-29 12:30:20 +00:00
kubaneko a4d183ce8e better peeling prototype 2023-03-28 18:18:51 +00:00
kubaneko c69efdb5f4 changed where iterations between linear portion and the next pow2 go 2023-03-27 15:59:53 +00:00
kubaneko 1a48dd6757 fixed byg 2023-03-15 21:09:45 +00:00
kubaneko 9d883347fd some minor divise fixes 2023-03-15 08:16:04 +00:00
kubaneko b9d5d2a699 fixed up fix-up functions 2023-03-15 00:21:44 +00:00
kubaneko 03453fc006 added param range 2023-03-14 17:23:00 +00:00
kubaneko e755feaf21 some refactoring and added lto-streaming for histograms 2023-03-13 21:31:03 +00:00
kubaneko 18ffdd8c9a maintaining histogram prototype 2023-03-07 20:09:11 +00:00
kubaneko 67511ca09d fixed error and improved peeling 2023-03-01 20:30:09 +00:00
kubaneko a19d946ebc change typo 2023-02-27 15:18:25 +00:00
kubaneko b2ac41c139 Added histogram params to libgcov 2023-02-26 19:17:42 +00:00
kubaneko 1cf1f75ee7 added primitive peeling and stuff 2023-02-23 23:17:37 +00:00
kubaneko 50eb12a4b3 removed hist_size 2023-02-23 23:17:36 +00:00
kubaneko 93b5586f6f corrected shift 2023-02-23 23:17:36 +00:00
kubaneko 98d59dcdeb fixed vector allocation 2023-02-23 23:17:36 +00:00
kubaneko 63142d74e3 changed vector from gc to heap 2023-02-23 23:17:36 +00:00
kubaneko 680d2bf08f vector for histogram_counters not working yet 2023-02-23 23:17:35 +00:00
kubaneko ecf3c41ca0 started with conversion to parametric histograms 2023-02-23 23:17:35 +00:00
kubaneko 81285bf85e new create_iv 2023-02-23 23:17:35 +00:00
kubaneko 223227bbe5 redo stuff 2023-02-23 23:17:34 +00:00
kubaneko bb8a802b81 fixed memory allocation for histogram_counters 2023-02-23 23:17:34 +00:00
kubaneko 1aa8abc667 added histogram_counters structure 2023-02-23 23:17:34 +00:00
kubaneko ef357765f5 temporary log fix 2023-02-23 23:17:34 +00:00
kubaneko 09ffc042ed messed with logarithms 2023-02-23 23:17:33 +00:00
kubaneko 8a387e2c25 added atomic counter 2023-02-23 23:17:33 +00:00
kubaneko b4bad98892 removed wild fprint removed redundant struct fields 2023-02-23 23:17:33 +00:00
kubaneko e44325a383 one histogram per loop, commit edges after deleting fake ones 2023-02-23 23:17:32 +00:00
kubaneko 158017f839 add loop_optimizer_init 2023-02-23 23:17:32 +00:00
kubaneko 258c38ee4b fixed loading found another bug 2023-02-23 23:17:32 +00:00
kubaneko 8196f7c7de squashed 2 commits
fixed function calls and tried to load histogram

tried to add loading and failed
2023-02-23 23:17:29 +00:00
kubaneko ffdd7efad3 fixed what I broke 2023-02-23 23:16:30 +00:00
Ondrej Kubanek 0509b471bd adding histogram for empty bbs 2023-02-23 23:16:30 +00:00
Ondrej Kubanek b0aaccee12 Revert "Revert "tried fix create_iv and failed""
This reverts commit da5b7fc0fdf0227aa8347ffab1568198ccc58c43.
2023-02-23 23:16:30 +00:00
Ondrej Kubanek 27a79341b7 deleted an assert to to test the if preheaders are needed in this case 2023-02-23 23:16:29 +00:00
Ondrej Kubanek fe7dd0a667 Revert "tried fix create_iv and failed"
This reverts commit 082a5ebc956de41baf9d45ed609542ddd520df2d.
2023-02-23 23:16:29 +00:00
Ondrej Kubanek 391f76e291 tried fix create_iv and failed 2023-02-23 23:16:29 +00:00
Ondrej Kubanek bc97895170 added myself to maintainers 2023-02-23 23:16:24 +00:00
Ondrej Kubanek 9b238f1505 Revert "test commit added myself to MAINTAINERS"
This reverts commit 79707034dca6941cf2b625f45ba0a08d3cd6753d.
2023-02-23 23:14:19 +00:00
Ondrej Kubanek 66ef904153 fixed logarithm 2023-02-23 23:13:44 +00:00
Ondrej Kubanek 773f935c43 loop counter prototype 2023-02-23 23:13:43 +00:00
Ondrej Kubanek ece91c91a9 test commit added myself to MAINTAINERS 2023-02-23 23:13:43 +00:00
Ondrej Kubanek e6f3908a2e added histogram profiler and added it to division 2023-02-23 23:13:43 +00:00
29 changed files with 1455 additions and 83 deletions
+2
View File
@@ -493,6 +493,7 @@ Matt Kraai <kraai@ftbfs.org>
Jan Kratochvil <jan.kratochvil@redhat.com>
Matthias Kretz <m.kretz@gsi.de>
Louis Krupp <louis.krupp@zoho.com>
Ondrej Kubanek <kubanek0ondrej@gmail.com>
Prathamesh Kulkarni <prathamesh.kulkarni@linaro.org>
Venkataramanan Kumar <venkataramanan.kumar@amd.com>
Doug Kwan <dougkwan@google.com>
@@ -724,6 +725,7 @@ Certificate of Origin Version 1.1. See https://gcc.gnu.org/dco.html for more
information.
Ondrej Kubanek <kubanek0ondrej@gmail.com>
Matthias Kretz <m.kretz@gsi.de>
Tim Lange <mail@tim-lange.me>
Jeff Law <jeffreyalaw@gmail.com>
+1
View File
@@ -1421,6 +1421,7 @@ OBJS = \
gimple-loop-interchange.o \
gimple-loop-jam.o \
gimple-loop-versioning.o \
gimple-loop-versioning-histograms.o \
gimple-low.o \
gimple-predicate-analysis.o \
gimple-pretty-print.o \
+219 -1
View File
@@ -199,6 +199,14 @@ flow_loop_free (class loop *loop)
}
ggc_free (loop->exits);
if (loop->counters)
{
va_heap::release (loop->counters->lin);
va_heap::release (loop->counters->exp);
if (loop->counters->mod)
va_heap::release (loop->counters->mod);
ggc_free (loop->counters);
}
ggc_free (loop);
}
@@ -1770,7 +1778,7 @@ loop_preheader_edge (const class loop *loop)
edge_iterator ei;
gcc_assert (loops_state_satisfies_p (LOOPS_HAVE_PREHEADERS)
&& ! loops_state_satisfies_p (LOOPS_MAY_HAVE_MULTIPLE_LATCHES));
&& !loops_state_satisfies_p (LOOPS_MAY_HAVE_MULTIPLE_LATCHES));
FOR_EACH_EDGE (e, ei, loop->header->preds)
if (e->src != loop->latch)
@@ -2181,3 +2189,213 @@ loops_list::walk_loop_tree (class loop *root, unsigned flags)
this->to_visit.quick_push (root->num);
}
unsigned int
hist_index (gcov_type_unsigned val, unsigned int lin_size, unsigned int exp_size)
{
unsigned int tot_size = lin_size + exp_size;
if (val < lin_size)
{
return val;
}
else
{
gcov_type_unsigned pow2 = floor_log2 (val);
gcov_type_unsigned lin_pow2 = floor_log2 (lin_size - 1);
if (lin_size < tot_size && pow2 == lin_pow2)
{
return lin_size;
}
if (tot_size > pow2 + ((lin_size - 1) - lin_pow2))
{
return pow2 + ((lin_size - 1) - lin_pow2);
}
else
{
return tot_size - 1;
}
}
}
void
histogram_counters_minus_upper_bound (histogram_counters *&hist_c,
gcov_type_unsigned difference)
{
if (difference == 0 || !hist_c)
return;
if (hist_c->sum == 0 && !hist_c->mod)
{
va_heap::release (hist_c->lin);
va_heap::release (hist_c->exp);
ggc_free (hist_c);
hist_c = NULL;
return;
}
auto &lin = *(hist_c->lin);
auto &exp = *(hist_c->exp);
hist_c->adjusted = true;
auto &sum = hist_c->sum;
unsigned int lin_size = lin.length ();
unsigned int exp_size = exp.length ();
// If the last linear counter does not contain other iterations
unsigned int i = 0;
for (; i < difference && i < lin.length (); ++i)
{
sum -= lin[i];
lin[i] = 0;
}
for (; i < lin.length (); ++i)
{
lin[i - difference] = lin[i];
lin[i] = 0;
}
lin.truncate (lin.length () > difference ? lin.length () - difference : 0);
// next pow2
gcov_type_unsigned pow2 = ((gcov_type_unsigned) 1)
<< (ceil_log2 (lin_size) + i + 1 - lin_size);
i = 0;
// we null all counters that cannot transfer to non-zero counts
for (; pow2 - 1 < difference && i < exp.length () - 1; ++i)
{
sum -= exp[0];
exp[0] = exp[i];
exp[i] = 0;
pow2 = pow2 << 1;
}
// if there are no more iterations we do not care
if (hist_c->sum == 0 && !hist_c->mod)
{
va_heap::release (hist_c->lin);
va_heap::release (hist_c->exp);
ggc_free (hist_c);
hist_c = NULL;
return;
}
// we want to change index 1/(1<<portion) of iterations
int portion = 1;
for (; i < exp.length () - 1 && portion < 10; ++i)
{
// we take a sample point and suppose by uniform distribution that all
// lesser iterations move same as this point
gcov_type_unsigned point = (pow2 >> 1) + (pow2 >> (1 + portion));
unsigned int ind
= point >= difference
? hist_index (point - difference, lin_size, exp_size) : 0;
while (ind == i)
{
// if nothing changes we decrease the portion of iteration changed
++portion;
point = (pow2 >> 1) + (pow2 >> (1 + portion));
ind = hist_index (point >= difference ? point - difference : 0, lin_size, exp_size);
}
int64_t diff = exp[i] / (1 << portion);
exp[ind > lin_size ? ind - lin_size : 0] += diff;
exp[i] -= diff;
pow2 <<= 1;
}
if (hist_c->mod)
{
auto &mod_h = *(hist_c->mod);
unsigned int mod_size = mod_h.length ();
// rotate the values by the difference
auto_vec<gcov_type> temp;
for (unsigned int j = 0; j < mod_size; ++j)
{
temp.safe_push (mod_h[j]);
}
for (unsigned int j = 0; j < mod_size; ++j)
{
mod_h[j] = temp[(mod_size + j - (difference % mod_size)) % mod_size];
}
}
}
void
histogram_counters_div_upper_bound (histogram_counters *&hist_c,
unsigned int divisor)
{
if (divisor < 2 || !hist_c)
return;
if (hist_c->sum == 0)
{
va_heap::release (hist_c->lin);
va_heap::release (hist_c->exp);
if (hist_c->mod)
va_heap::release (hist_c->mod);
ggc_free (hist_c);
hist_c = NULL;
return;
}
auto &lin = *(hist_c->lin);
auto &exp = *(hist_c->exp);
hist_c->adjusted = true;
unsigned int lin_size = lin.length ();
unsigned int exp_size = exp.length ();
unsigned int tot_size = lin_size + exp_size;
unsigned int i = 1;
for (; i < lin.length (); i++)
{
// we want to take the roof of the division
lin[i / divisor + MIN (1, i % divisor)] += lin[i];
lin[i] = 0;
}
lin.truncate (lin.length () ? (lin.length () - 1) / divisor
+ MIN (1, (lin.length () - 1) % divisor) + 1
: 0);
for (; i < tot_size - 1; i++)
{
gcov_type_unsigned upper_pow2
= ((gcov_type_unsigned) 1) << (ceil_log2 (lin_size) + i + 1 - lin_size);
gcov_type_unsigned half = (upper_pow2 >> 2) + (upper_pow2 >> 1);
unsigned int ind = hist_index (half / divisor, lin_size, exp_size);
// hist is allways different then i since we know divisor>1
exp[ind > lin_size ? ind - lin_size : 0] += exp[i];
exp[i] = 0;
}
// we do not try to maintain modulo info when dividing since it is often
// impossible
// TODO we can sometimes save this if the divisor and the mod_size are
// co-prime
if (hist_c->mod)
{
va_heap::release (hist_c->mod);
hist_c->mod = NULL;
}
}
// adjust estimates after peeling npeel times
// also used in copy header optimization
void
adjust_loop_estimates_minus (class loop *loop, unsigned HOST_WIDE_INT npeel,
bool precise)
{
if (loop->any_estimate)
{
if (wi::leu_p (npeel, loop->nb_iterations_estimate))
loop->nb_iterations_estimate -= npeel;
else
loop->any_estimate = false;
}
if (loop->any_upper_bound && precise)
{
if (wi::ltu_p (npeel, loop->nb_iterations_upper_bound))
loop->nb_iterations_upper_bound -= npeel;
else
loop->nb_iterations_upper_bound = 0;
}
if (loop->counters)
{
histogram_counters_minus_upper_bound (loop->counters, npeel);
}
if (loop->any_likely_upper_bound)
{
if (wi::ltu_p (npeel, loop->nb_iterations_likely_upper_bound))
loop->nb_iterations_likely_upper_bound -= npeel;
else
{
loop->any_estimate = true;
loop->nb_iterations_estimate = 0;
loop->nb_iterations_likely_upper_bound = 0;
}
}
}
+24
View File
@@ -92,6 +92,17 @@ struct loop_exit_hasher : ggc_ptr_hash<loop_exit>
static void remove (loop_exit *);
};
// Information about histogram of the loop from profiling
struct GTY (()) histogram_counters
{
bool adjusted;
gcov_type sum;
vec<gcov_type, va_heap, vl_embed> *GTY ((skip)) lin;
vec<gcov_type, va_heap, vl_embed> *GTY ((skip)) exp;
vec<gcov_type, va_heap, vl_embed> *GTY ((skip)) mod;
};
typedef class loop *loop_p;
/* An integer estimation of the number of iterations. Estimate_state
@@ -113,6 +124,8 @@ struct GTY ((chain_next ("%h.next"))) control_iv {
struct control_iv *next;
};
#include "value-prof.h"
/* Structure to hold information for each natural loop. */
class GTY ((chain_next ("%h.next"))) loop {
public:
@@ -272,6 +285,8 @@ public:
the basic-block from being collected but its index can still be
reused. */
basic_block former_header;
histogram_counters *counters;
};
/* Set if the loop is known to be infinite. */
@@ -916,6 +931,15 @@ extern bool get_estimated_loop_iterations (class loop *loop, widest_int *nit);
extern bool get_max_loop_iterations (const class loop *loop, widest_int *nit);
extern bool get_likely_max_loop_iterations (class loop *loop, widest_int *nit);
extern int bb_loop_depth (const_basic_block);
extern void
histogram_counters_minus_upper_bound (histogram_counters *&hist_c,
gcov_type_unsigned difference);
extern void
histogram_counters_div_upper_bound (histogram_counters *&hist_c,
unsigned int divisor);
extern void
adjust_loop_estimates_minus (class loop *loop, unsigned HOST_WIDE_INT npeel,
bool precise);
/* Converts VAL to widest_int. */
+48 -5
View File
@@ -959,6 +959,16 @@ copy_loop_info (class loop *loop, class loop *target)
target->finite_p = loop->finite_p;
target->unroll = loop->unroll;
target->owned_clique = loop->owned_clique;
if (loop->counters)
{
target->counters = ggc_alloc<histogram_counters> ();
target->counters->sum = loop->counters->sum;
target->counters->adjusted = loop->counters->adjusted;
target->counters->lin = vec_safe_copy (loop->counters->lin);
target->counters->exp = vec_safe_copy (loop->counters->exp);
if (loop->counters->mod)
target->counters->mod = vec_safe_copy (loop->counters->mod);
}
}
/* Copies copy of LOOP as subloop of TARGET loop, placing newly
@@ -1148,11 +1158,44 @@ duplicate_loop_body_to_header_edge (class loop *loop, edge e,
}
scale_step = XNEWVEC (profile_probability, ndupl);
for (i = 1; i <= ndupl; i++)
scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
? prob_pass_wont_exit
: prob_pass_thru;
// If we have histogram then we know precise probabilities for peeling
// is_latch means unrolling
if (!loop->counters || is_latch)
{
for (i = 1; i <= ndupl; i++)
scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
? prob_pass_wont_exit
: prob_pass_thru;
}
else
{
gcov_type psum = 0;
gcov_type sum = loop->counters->sum;
for (i = 0;
i < ndupl && i < (unsigned int) loop->counters->lin->length ()
&& sum != psum;
i++)
{
scale_step[i] = profile_probability::always ()
- ((*loop->counters->lin)[i]
? profile_probability::always ()
/ (loop->counters->sum - psum)
* (*loop->counters->lin)[i]
: profile_probability::never ());
// we set whether it was adjusted or is still precise
if (loop->counters->adjusted)
{
scale_step[i].adjusted ();
}
psum += (*loop->counters->lin)[i];
}
++i;
for (; i <= ndupl; i++)
scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
? prob_pass_wont_exit
: prob_pass_thru;
// should rescale
}
/* Complete peeling is special as the probability of exit in last
copy becomes 1. */
+1
View File
@@ -59,5 +59,6 @@ class loop * loop_version (class loop *, void *,
basic_block *,
profile_probability, profile_probability,
profile_probability, profile_probability, bool);
void adjust_loop_info_after_peeling (class loop *loop, int npeel, bool precise);
#endif /* GCC_CFGLOOPMANIP_H */
+16
View File
@@ -2291,6 +2291,14 @@ fpeel-loops
Common Var(flag_peel_loops) Optimization
Perform loop peeling.
fuse-histograms-in-peeling
Common Var(flag_use_histogram_in_peeling) Init(1) Optimization
Use histograms in loop peeling heuristics.
fpeel-loops-without-histogram
Common Var(flag_peel_loops_without_histogram) Init(1) Optimization
Peel also loop with no loop histogram.
fpeephole
Common Var(flag_no_peephole,0) Optimization
Enable machine specific peephole optimizations.
@@ -2388,6 +2396,10 @@ fprofile-reproducible=
Common Joined RejectNegative Var(flag_profile_reproducible) Enum(profile_reproducibility) Init(PROFILE_REPRODUCIBILITY_SERIAL)
-fprofile-reproducible=[serial|parallel-runs|multithreaded] Control level of reproducibility of profile gathered by -fprofile-generate.
fprofile-loops
Common Var(flag_profile_loops) Init(1) Optimization
Profile loop histograms.
Enum
Name(profile_update) Type(enum profile_update) UnknownError(unknown profile update method %qs)
@@ -3099,6 +3111,10 @@ fversion-loops-for-strides
Common Var(flag_version_loops_for_strides) Optimization
Version loops based on whether indices have a stride of one.
fversion-loops-using-histograms
Common Var(flag_version_loops_using_histograms) Optimization
Version loops based on iteration count histograms
funwind-tables
Common Var(flag_unwind_tables) Optimization
Just generate unwind tables for exception handling.
+4
View File
@@ -49,3 +49,7 @@ DEF_GCOV_COUNTER(GCOV_COUNTER_IOR, "ior", _ior)
/* Time profile collecting first run of a function */
DEF_GCOV_COUNTER(GCOV_TIME_PROFILER, "time_profiler", _time_profile)
/* Histogram of values from 1 to 8 and them powers of 2 */
DEF_GCOV_COUNTER(GCOV_COUNTER_V_HISTOGRAM, "histogram", _add)
+264
View File
@@ -0,0 +1,264 @@
/* Loop versioning using histograms.
Copyright (C) 2023 Free Software Foundation, Inc.
Contributed by Ondrej Kubanek and Jan Hubicka
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3, or (at your option) any
later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "tree.h"
#include "gimple.h"
#include "gimple-iterator.h"
#include "tree-pass.h"
#include "gimplify-me.h"
#include "cfgloop.h"
#include "tree-ssa-loop.h"
#include "ssa.h"
#include "tree-scalar-evolution.h"
#include "tree-ssa-loop-ivopts.h"
#include "fold-const.h"
#include "tree-ssa-propagate.h"
#include "tree-inline.h"
#include "domwalk.h"
#include "tree-vectorizer.h"
#include "omp-general.h"
#include "predict.h"
#include "tree-into-ssa.h"
#include "gimple-range.h"
#include "tree-cfg.h"
#include "tree-ssa-loop-niter.h"
#include "tree-pretty-print.h"
namespace {
/* Histogram loop versioning pass.
The pass looks for loops with single exit conditioned by a induction
variable test. If histogram indicates that such loop iterates usually
N times, we produce a versined version:
if (number_of_iterations == N)
for (int i = 0; i < N; i++)
loop_body;
else
original_loop.
After this transformation the specialized version can often be
fully unrolled or vectorized. */
const pass_data pass_data_loop_histogram_versioning =
{
GIMPLE_PASS, /* type */
"hversion", /* name */
OPTGROUP_LOOP, /* optinfo_flags */
TV_LOOP_VERSIONING, /* tv_id */
PROP_cfg, /* properties_required */
0, /* properties_provided */
0, /* properties_destroyed */
0, /* todo_flags_start */
0, /* todo_flags_finish */
};
class pass_loop_histogram_versioning : public gimple_opt_pass
{
public:
pass_loop_histogram_versioning (gcc::context *ctxt)
: gimple_opt_pass (pass_data_loop_histogram_versioning, ctxt)
{}
/* opt_pass methods: */
bool gate (function *) final override
{
return flag_version_loops_using_histograms;
}
unsigned int execute (function *) final override;
};
unsigned int
pass_loop_histogram_versioning::execute (function *fn)
{
bool changed = false;
if (number_of_loops (fn) <= 1)
return 0;
loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
scev_initialize ();
estimate_numbers_of_iterations (cfun);
for (auto loop : loops_list (cfun, 0))
{
/* We version only inner loops optimized for speed with histogram
attachd. */
if (loop->inner != NULL
|| loop->counters == NULL
|| !optimize_loop_for_speed_p (loop))
continue;
int best_iters = - 1;
gcov_type best_val = 0;
/* Look for dominating iteration count. */
for (int i = 0; i < loop->counters->lin->length (); i++)
{
if ((*(loop->counters->lin))[i] > best_val)
{
best_val = (*(loop->counters->lin))[i];
best_iters = i;
}
}
if (best_val <= 0 || best_val < loop->counters->sum * param_loop_versioning_histogram_prcnt / 100)
continue;
if (dump_file)
fprintf (dump_file, "Loop %i has dominating number of iterations %i\n",
loop->num, best_iters);
if (!best_iters)
{
if (dump_file)
fprintf (dump_file,
"Not versioning since normal peeling will do the job\n");
continue;
}
/* If loop has no single exit, don't try to version.
In this case we don not know if the loop iteration count corresponds
to the IV test.
TODO: We may be able to also handle loop with single dominating
exit. */
auto_vec<edge> exits = get_loop_exit_edges (loop);
if (exits.length () != 1)
{
if (dump_file)
fprintf (dump_file, "Wrong number of exits: %i\n", exits.length ());
continue;
}
/* Analyze number of iterations. */
class tree_niter_desc niter;
if (!number_of_iterations_exit (loop, exits[0],
&niter, false, false))
{
if (dump_file)
fprintf (dump_file, "niter analysis failed\n");
continue;
}
/* If the loop exit is not simple IV test or the loop already
has known number of iterations, give up. */
if (!niter.niter
|| !integer_onep (niter.assumptions)
|| TREE_CODE (niter.niter) == INTEGER_CST)
{
if (dump_file)
{
if (TREE_CODE (niter.niter) == INTEGER_CST)
fprintf (dump_file, "niter is already constant:");
else
fprintf (dump_file, "niter wrong:");
print_generic_expr (dump_file, niter.niter);
fprintf (dump_file, "\n");
}
continue;
}
/* Sanity check with maximal number of iterations. */
int maxiter = max_loop_iterations_int (loop);
if (maxiter >= 0 && maxiter < best_iters)
{
if (dump_file)
fprintf (dump_file, "Upper bound on number of iterations is too small\n");
continue;
}
/* Compute loop body size. */
int ninsns = 0;
basic_block *bbs = get_loop_body (loop);
for (unsigned int i = 0; i < loop->num_nodes; i++)
for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
!gsi_end_p (gsi) && ninsns <= param_loop_versioning_max_inner_insns;
gsi_next (&gsi))
ninsns += estimate_num_insns (gsi_stmt (gsi), &eni_size_weights);
free (bbs);
if (ninsns > param_loop_versioning_max_inner_insns)
{
if (dump_file)
fprintf (dump_file, "--param loop-versioning-max-inner-insns exceeded\n");
continue;
}
/* Do versining. */
dump_user_location_t locus = last_stmt (exits[0]->src);
basic_block cond_bb;
gimple_seq stmts = NULL;
initialize_original_copy_tables ();
tree op = force_gimple_operand_1
(fold_build2 (NE_EXPR, boolean_type_node,
niter.niter,
build_int_cst (TREE_TYPE (niter.niter),
best_iters)),
&stmts, is_gimple_condexpr_for_cond, NULL_TREE);
class loop *optimized_loop
= loop_version (loop, op, &cond_bb,
/* It may be more precise to take probability from
histogram, but it also may lead for
over-specialization for the train run. */
profile_probability::very_unlikely (),
profile_probability::very_likely (),
profile_probability::very_unlikely (),
profile_probability::very_likely (), true);
free_original_copy_tables ();
/* Insert IV test. */
if (stmts)
{
gimple_stmt_iterator gsi = gsi_last_bb (cond_bb);
gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
}
/* Update optimized loop.
We force it to have given number of iteraitons and thus
it makes sense to drop histogram counters. */
record_niter_bound (optimized_loop, best_val, false, true);
optimized_loop->counters = 0;
auto_vec<edge> optimized_exits = get_loop_exit_edges (optimized_loop);
create_canonical_iv (optimized_loop, optimized_exits[0],
build_int_cst (integer_type_node, best_iters));
/* Update counters of non-optiized loop.
TODO: also recompute estimated number of itraitons
instead of droping it bellow. */
(*(loop->counters->lin))[best_iters] = 0;
loop->counters->sum -= best_val;
loop->any_estimate = false;
changed = true;
if (dump_enabled_p ())
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | TDF_DETAILS, locus,
"loop %i versioned based on histogram "
"for %i iterations (header execution count %d)\n",
loop->num, best_iters,
optimized_loop->header->count.to_gcov_type ());
}
free_numbers_of_iterations_estimates (cfun);
scev_finalize ();
loop_optimizer_finalize ();
return (changed ? TODO_update_ssa : 0) | TODO_cleanup_cfg;
}
} // anon namespace
gimple_opt_pass *
make_pass_loop_histogram_versioning (gcc::context *ctxt)
{
return new pass_loop_histogram_versioning (ctxt);
}
+3
View File
@@ -41,6 +41,8 @@ along with GCC; see the file COPYING3. If not see
#include "tree-into-ssa.h"
#include "gimple-range.h"
#include "tree-cfg.h"
#include "tree-ssa-loop-niter.h"
#include "tree-pretty-print.h"
namespace {
@@ -1808,3 +1810,4 @@ make_pass_loop_versioning (gcc::context *ctxt)
{
return new pass_loop_versioning (ctxt);
}
+21
View File
@@ -533,6 +533,10 @@ unroll_loop_constant_iterations (class loop *loop)
desc->noloop_assumptions = NULL_RTX;
desc->niter -= exit_mod;
loop->nb_iterations_upper_bound -= exit_mod;
if (loop->counters)
{
histogram_counters_minus_upper_bound (loop->counters, exit_mod);
}
if (loop->any_estimate
&& wi::leu_p (exit_mod, loop->nb_iterations_estimate))
loop->nb_iterations_estimate -= exit_mod;
@@ -578,6 +582,11 @@ unroll_loop_constant_iterations (class loop *loop)
desc->niter -= exit_mod + 1;
loop->nb_iterations_upper_bound -= exit_mod + 1;
if (loop->counters)
{
histogram_counters_minus_upper_bound (loop->counters,
exit_mod + 1);
}
if (loop->any_estimate
&& wi::leu_p (exit_mod + 1, loop->nb_iterations_estimate))
loop->nb_iterations_estimate -= exit_mod + 1;
@@ -632,6 +641,10 @@ unroll_loop_constant_iterations (class loop *loop)
desc->niter /= max_unroll + 1;
loop->nb_iterations_upper_bound
= wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);
if (loop->counters)
{
histogram_counters_div_upper_bound (loop->counters, exit_mod);
}
if (loop->any_estimate)
loop->nb_iterations_estimate
= wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);
@@ -1095,6 +1108,10 @@ unroll_loop_runtime_iterations (class loop *loop)
gen_int_mode (max_unroll + 1, desc->mode));
loop->nb_iterations_upper_bound
= wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);
if (loop->counters)
{
histogram_counters_div_upper_bound (loop->counters, max_unroll + 1);
}
if (loop->any_estimate)
loop->nb_iterations_estimate
= wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);
@@ -1107,6 +1124,10 @@ unroll_loop_runtime_iterations (class loop *loop)
simplify_gen_binary (MINUS, desc->mode, desc->niter_expr, const1_rtx);
desc->noloop_assumptions = NULL_RTX;
--loop->nb_iterations_upper_bound;
if (loop->counters)
{
histogram_counters_minus_upper_bound (loop->counters, 1);
}
if (loop->any_estimate
&& loop->nb_iterations_estimate != 0)
--loop->nb_iterations_estimate;
+32
View File
@@ -1126,6 +1126,38 @@ input_cfg (class lto_input_block *ib, class data_in *data_in,
loop->any_estimate = streamer_read_hwi (ib);
if (loop->any_estimate)
loop->nb_iterations_estimate = streamer_read_widest_int (ib);
// whether the loop has histogram counter
if (streamer_read_hwi (ib))
{
loop->counters = ggc_alloc<histogram_counters> ();
loop->counters->sum = streamer_read_gcov_count (ib);
loop->counters->lin = NULL;
loop->counters->exp = NULL;
loop->counters->mod = NULL;
int lin_len = streamer_read_hwi (ib);
int exp_len = streamer_read_hwi (ib);
int mod_len = streamer_read_hwi (ib);
vec_safe_grow (loop->counters->lin, lin_len);
vec_safe_grow (loop->counters->exp, exp_len);
for (int i = 0; i < lin_len; ++i)
{
(*loop->counters->lin)[i] = streamer_read_gcov_count (ib);
}
for (int i = 0; i < exp_len; ++i)
{
(*loop->counters->exp)[i] = streamer_read_gcov_count (ib);
}
// do we have modulos
if (mod_len)
{
vec_safe_grow (loop->counters->mod,
mod_len);
for (int i = 0; i < mod_len; ++i)
{
(*loop->counters->mod)[i] = streamer_read_gcov_count (ib);
}
}
}
/* Read OMP SIMD related info. */
loop->safelen = streamer_read_hwi (ib);
+25
View File
@@ -2181,6 +2181,31 @@ output_cfg (struct output_block *ob, struct function *fn)
streamer_write_hwi (ob, loop->any_estimate);
if (loop->any_estimate)
streamer_write_widest_int (ob, loop->nb_iterations_estimate);
streamer_write_hwi (ob, loop->counters != NULL);
if (loop->counters)
{
streamer_write_gcov_count (ob, loop->counters->sum);
streamer_write_hwi (ob, loop->counters->lin->length ());
streamer_write_hwi (ob, loop->counters->exp->length ());
streamer_write_hwi (ob, loop->counters->mod ? loop->counters->mod->length () : 0);
for (unsigned int i = 0; i < loop->counters->lin->length (); ++i)
{
streamer_write_gcov_count (ob, (*loop->counters->lin)[i]);
}
for (unsigned int i = 0;
i < (unsigned int) loop->counters->exp->length (); ++i)
{
streamer_write_gcov_count (ob, (*loop->counters->exp)[i]);
}
if (loop->counters->mod)
{
for (unsigned int i = 0;
i < (unsigned int) loop->counters->mod->length (); ++i)
{
streamer_write_gcov_count (ob, (*loop->counters->mod)[i]);
}
}
}
/* Write OMP SIMD related info. */
streamer_write_hwi (ob, loop->safelen);
+1
View File
@@ -2044,6 +2044,7 @@ enable_fdo_optimizations (struct gcc_options *opts,
SET_OPTION_IF_UNSET (opts, opts_set, flag_loop_interchange, value);
SET_OPTION_IF_UNSET (opts, opts_set, flag_unroll_jam, value);
SET_OPTION_IF_UNSET (opts, opts_set, flag_tree_loop_distribution, value);
SET_OPTION_IF_UNSET (opts, opts_set, flag_version_loops_using_histograms, value);
}
/* -f{,no-}sanitize{,-recover}= suboptions. */
+24
View File
@@ -426,6 +426,10 @@ The maximum number of instructions in an inner loop that is being considered for
Common Joined UInteger Var(param_loop_versioning_max_outer_insns) Init(100) Param Optimization
The maximum number of instructions in an outer loop that is being considered for versioning, on top of the instructions in inner loops.
-param=loop-versioning-histogram_prcnt=
Common Joined UInteger Var(param_loop_versioning_histogram_prcnt) Init(30) Param Optimization
Percentage of the dominating iteration count required for versioning.
-param=lra-inheritance-ebb-probability-cutoff=
Common Joined UInteger Var(param_lra_inheritance_ebb_probability_cutoff) Init(40) IntegerRange(0, 100) Param Optimization
Minimal fall-through edge probability in percentage used to add BB to inheritance EBB in LRA.
@@ -874,6 +878,26 @@ Min. ratio of insns to mem ops to enable prefetching in a loop.
Common Joined UInteger Var(param_prefetch_minimum_stride) Init(-1) Param Optimization
The minimum constant stride beyond which we should use prefetch hints for.
-param=profile-histogram-size-exp=
Common Joined UInteger Var(param_profile_histogram_size_exp) Init(1) IntegerRange(1, 255) Param Optimization
Size of additional counters in exponential intervals after the linear section.
-param=profile-histogram-size-mod=
Common Joined UInteger Var(param_profile_histogram_size_mod) Init(32) IntegerRange(1, 255) Param Optimization
Counts the modulo of loop iterations by its size.
-param=profile-histogram-size-lin=
Common Joined UInteger Var(param_profile_histogram_size_lin) Init(16) IntegerRange(1, 255) Param Optimization
Size of the linear portion of the histogram counter for profile feedback.
-param=profile-histogram-peel-prcnt=
Common Joined UInteger Var(param_profile_histogram_peel_prcnt) Init(6) Param Optimization
Peels if it can eliminate at least this percentage per 1 peeling.
-param=profile-histogram-peel-overall-prcnt=
Common Joined UInteger Var(param_profile_histogram_peel_overall_prcnt) Init(30) Param Optimization
Peels if it can eliminate at least this percentage of executions of the loop.
-param=profile-func-internal-id=
Common Joined UInteger Var(param_profile_func_internal_id) IntegerRange(0, 1) Param
Use internal function id in profile lookup.
+2
View File
@@ -229,6 +229,7 @@ along with GCC; see the file COPYING3. If not see
NEXT_PASS (pass_merge_phi);
NEXT_PASS (pass_phiopt, false /* early_p */);
NEXT_PASS (pass_tail_recursion);
NEXT_PASS (pass_loop_histogram_versioning);
NEXT_PASS (pass_ch);
NEXT_PASS (pass_lower_complex);
NEXT_PASS (pass_sra);
@@ -275,6 +276,7 @@ along with GCC; see the file COPYING3. If not see
/* Before loop_init we rewrite no longer addressed locals into SSA
form if possible. */
NEXT_PASS (pass_tree_loop_init);
//NEXT_PASS (pass_loop_histogram_versioning);
NEXT_PASS (pass_tree_unswitch);
NEXT_PASS (pass_scev_cprop);
NEXT_PASS (pass_loop_split);
+70 -4
View File
@@ -66,6 +66,7 @@ along with GCC; see the file COPYING3. If not see
#include "cfgloop.h"
#include "sreal.h"
#include "file-prefix-map.h"
#include "tree-scalar-evolution.h"
#include "profile.h"
@@ -189,6 +190,10 @@ instrument_values (histogram_values values)
gimple_gen_time_profiler (t);
break;
case HIST_TYPE_HISTOGRAM:
gimple_gen_histogram_profiler (hist, t);
break;
default:
gcc_unreachable ();
}
@@ -915,8 +920,63 @@ compute_value_histograms (histogram_values values, unsigned cfg_checksum,
bool topn_p = (hist->type == HIST_TYPE_TOPN_VALUES
|| hist->type == HIST_TYPE_INDIR_CALL);
/* Histogram profiler counter is not related to any statement,
so that we have to read the counter and set the value to
the corresponding loop */
if (hist->type == HIST_TYPE_HISTOGRAM)
{
auto lp = hist->hvalue.lp;
if (act_count[t])
{
lp->counters = ggc_alloc<histogram_counters> ();
gcov_type sum = 0;
unsigned int tot_size = param_profile_histogram_size_lin
+ param_profile_histogram_size_exp;
lp->counters->adjusted = false;
lp->counters->lin = NULL;
lp->counters->exp = NULL;
// iteration vector
vec_safe_grow_cleared (lp->counters->lin,
param_profile_histogram_size_lin);
vec_safe_grow_cleared (lp->counters->exp,
param_profile_histogram_size_exp);
for (int i = 0; i < (int) param_profile_histogram_size_lin; ++i)
{
auto hst = lp->counters->lin;
(*hst)[i] = act_count[t][i];
sum += act_count[t][i];
}
for (int i = 0; i < (int) param_profile_histogram_size_exp; ++i)
{
auto hst = lp->counters->exp;
(*hst)[i] = act_count[t][i + param_profile_histogram_size_lin];
sum += act_count[t][i + param_profile_histogram_size_lin];
}
lp->counters->sum = sum;
// modular vector
lp->counters->mod = NULL;
vec_safe_grow_cleared (lp->counters->mod,
param_profile_histogram_size_mod);
for (int j = 0; j < param_profile_histogram_size_mod; ++j)
{
auto mod = lp->counters->mod;
(*mod)[j] = act_count[t][j + tot_size];
}
if (sum == 0)
{
va_heap::release (lp->counters->lin);
va_heap::release (lp->counters->exp);
va_heap::release (lp->counters->mod);
ggc_free (lp->counters);
lp->counters = NULL;
}
act_count[t] += hist->n_counters;
}
}
/* TOP N counter uses variable number of counters. */
if (topn_p)
else if (topn_p)
{
unsigned total_size;
if (act_count[t])
@@ -1176,6 +1236,10 @@ branch_prob (bool thunk)
total_num_times_called++;
calculate_dominance_info (CDI_DOMINATORS);
loop_optimizer_init (LOOPS_NORMAL);
scev_initialize ();
flow_call_edges_add (NULL);
add_noreturn_fake_exit_edges ();
@@ -1528,16 +1592,16 @@ branch_prob (bool thunk)
if (flag_profile_values)
instrument_values (values);
/* Commit changes done by instrumentation. */
gsi_commit_edge_inserts ();
}
/* Commit changes done by instrumentation. */
gsi_commit_edge_inserts ();
free_aux_for_edges ();
values.release ();
free_edge_list (el);
coverage_end_function (lineno_checksum, cfg_checksum);
scev_finalize ();
if (flag_branch_probabilities
&& (profile_status_for_fn (cfun) == PROFILE_READ))
{
@@ -1556,6 +1620,8 @@ branch_prob (bool thunk)
}
compute_function_frequency ();
}
loop_optimizer_finalize ();
free_dominance_info (CDI_DOMINATORS);
}
/* Union find algorithm implementation for the basic blocks using
+1
View File
@@ -373,6 +373,7 @@ extern gimple_opt_pass *make_pass_tree_loop (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_tree_no_loop (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_tree_loop_init (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_loop_versioning (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_loop_histogram_versioning (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_lim (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_linterchange (gcc::context *ctxt);
extern gimple_opt_pass *make_pass_tree_unswitch (gcc::context *ctxt);
+46
View File
@@ -52,16 +52,21 @@ along with GCC; see the file COPYING3. If not see
#include "stringpool.h"
#include "attribs.h"
#include "tree-pretty-print.h"
#include "gimple-pretty-print.h"
#include "langhooks.h"
#include "stor-layout.h"
#include "xregex.h"
#include "alloc-pool.h"
#include "symbol-summary.h"
#include "symtab-thunks.h"
#include "cfgloop.h"
#include "print-tree.h"
static GTY(()) tree gcov_type_node;
static GTY(()) tree tree_interval_profiler_fn;
static GTY(()) tree tree_pow2_profiler_fn;
static GTY (()) tree tree_histogram_profiler_fn;
static GTY(()) tree tree_topn_values_profiler_fn;
static GTY(()) tree tree_indirect_call_profiler_fn;
static GTY(()) tree tree_average_profiler_fn;
@@ -119,6 +124,7 @@ gimple_init_gcov_profiler (void)
{
tree interval_profiler_fn_type;
tree pow2_profiler_fn_type;
tree histogram_profiler_fn_type;
tree topn_values_profiler_fn_type;
tree gcov_type_ptr;
tree ic_profiler_fn_type;
@@ -161,6 +167,19 @@ gimple_init_gcov_profiler (void)
= tree_cons (get_identifier ("leaf"), NULL,
DECL_ATTRIBUTES (tree_pow2_profiler_fn));
/* void (*) (gcov_type *, gcov_type) */
histogram_profiler_fn_type
= build_function_type_list (void_type_node, gcov_type_ptr,
gcov_type_node, gcov_type_node, NULL_TREE);
fn_name = concat ("__gcov_histogram_profiler", fn_suffix, NULL);
tree_histogram_profiler_fn
= build_fn_decl (fn_name, histogram_profiler_fn_type);
free (CONST_CAST (char *, fn_name));
TREE_NOTHROW (tree_histogram_profiler_fn) = 1;
DECL_ATTRIBUTES (tree_histogram_profiler_fn)
= tree_cons (get_identifier ("leaf"), NULL,
DECL_ATTRIBUTES (tree_histogram_profiler_fn));
/* void (*) (gcov_type *, gcov_type) */
topn_values_profiler_fn_type
= build_function_type_list (void_type_node,
@@ -228,6 +247,7 @@ gimple_init_gcov_profiler (void)
late, we need to initialize them by hand. */
DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
DECL_ASSEMBLER_NAME (tree_histogram_profiler_fn);
DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
@@ -336,6 +356,32 @@ gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
gsi_insert_before (&gsi, call, GSI_NEW_STMT);
}
void
gimple_gen_histogram_profiler (histogram_value value,
unsigned tag) // , edge_def* edge
{
gimple *stmt = value->hvalue.stmt;
gcc_assert (!stmt);
auto lp = value->hvalue.lp;
gcc_assert (lp);
tree ref_ptr = tree_coverage_counter_addr (tag, 0);
tree hist_size = build_int_cst_type (
gcov_type_node, param_profile_histogram_size_lin
| (gcov_type (param_profile_histogram_size_exp) << 10)
| (gcov_type (param_profile_histogram_size_mod) << 20));
gcall *call;
auto_vec<edge> exits = get_loop_exit_edges (lp);
for (auto exit : exits)
{
if (!(exit->flags & (EDGE_COMPLEX | EDGE_FAKE)))
{
call = gimple_build_call (tree_histogram_profiler_fn, 3, ref_ptr,
value->hvalue.value, hist_size);
gsi_insert_seq_on_edge (exit, call);
}
}
}
/* Output instructions as GIMPLE trees for code to find the most N common
values. VALUE is the expression whose value is profiled. TAG is the tag
of the section for counters. */
+119 -12
View File
@@ -381,7 +381,7 @@ unsigned int
ch_base::copy_headers (function *fun)
{
basic_block header;
edge exit, entry;
edge exit, nonexit, entry;
basic_block *bbs, *copied_bbs;
unsigned n_bbs;
unsigned bbs_size;
@@ -453,8 +453,16 @@ ch_base::copy_headers (function *fun)
the header to have just a single successor and copying up to
postdominator. */
exit = NULL;
nonexit = NULL;
n_bbs = 0;
int nexits = 0;
profile_count exit_count = profile_count::zero ();
profile_count entry_count = profile_count::zero ();
edge e;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, loop->header->preds)
if (e->src != loop->latch)
entry_count += e->count ();
while (should_duplicate_loop_header_p (header, loop, &remaining_limit))
{
if (dump_file && (dump_flags & TDF_DETAILS))
@@ -463,15 +471,23 @@ ch_base::copy_headers (function *fun)
/* Find a successor of header that is inside a loop; i.e. the new
header after the condition is copied. */
if (flow_bb_inside_loop_p (loop, EDGE_SUCC (header, 0)->dest))
exit = EDGE_SUCC (header, 0);
{
nonexit = EDGE_SUCC (header, 0);
exit = EDGE_SUCC (header, 1);
}
else
exit = EDGE_SUCC (header, 1);
{
nonexit = EDGE_SUCC (header, 1);
exit = EDGE_SUCC (header, 0);
}
exit_count += exit->count ();
nexits++;
bbs[n_bbs++] = header;
gcc_assert (bbs_size > n_bbs);
header = exit->dest;
header = nonexit->dest;
}
if (!exit)
if (!nonexit)
continue;
if (dump_file && (dump_flags & TDF_DETAILS))
@@ -483,8 +499,11 @@ ch_base::copy_headers (function *fun)
/* Ensure that the header will have just the latch as a predecessor
inside the loop. */
if (!single_pred_p (exit->dest))
exit = single_pred_edge (split_edge (exit));
if (!single_pred_p (nonexit->dest))
{
header = split_edge (nonexit);
exit = single_pred_edge (header);
}
entry = loop_preheader_edge (loop);
@@ -542,10 +561,20 @@ ch_base::copy_headers (function *fun)
}
}
/* Ensure that the latch and the preheader is simple (we know that they
are not now, since there was the loop exit condition. */
split_edge (loop_preheader_edge (loop));
split_edge (loop_latch_edge (loop));
/* Update header of the loop. */
loop->header = header;
/* Find correct latch. We only duplicate chain of conditionals so
there should be precisely two edges to the new header. One entry
edge and one to latch. */
FOR_EACH_EDGE (e, ei, loop->header->preds)
if (header != e->src)
{
loop->latch = e->src;
break;
}
/* Ensure that the latch is simple. */
if (!single_succ_p (loop_latch_edge (loop)->src))
split_edge (loop_latch_edge (loop));
if (dump_file && (dump_flags & TDF_DETAILS))
{
@@ -554,8 +583,86 @@ ch_base::copy_headers (function *fun)
else
fprintf (dump_file, "Loop %d is still not do-while loop.\n",
loop->num);
fprintf (dump_file, "Exit count: ");
exit_count.dump (dump_file);
fprintf (dump_file, "\nEntry count: ");
entry_count.dump (dump_file);
fprintf (dump_file, "\n");
}
/* We possibly decreased number of itrations by 1. */
auto_vec<edge> exits = get_loop_exit_edges (loop);
bool precise = (nexits == (int) exits.length ());
if (!get_max_loop_iterations_int (loop))
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Loop %d no longer loops.\n", loop->num);
/* TODO: We can unloop like in tree-ssa-loop-ivcanon. */
precise = false;
}
/* Check that loop may not terminate in other way than via
basic blocks we duplicated. */
if (precise)
{
basic_block *bbs = get_loop_body (loop);
for (unsigned i = 0; i < loop->num_nodes && precise; ++i)
{
basic_block bb = bbs[i];
bool found_exit = false;
FOR_EACH_EDGE (e, ei, bb->succs)
if (!flow_bb_inside_loop_p (loop, e->dest))
{
found_exit = true;
break;
}
/* If BB has exit, it was duplicated. */
if (found_exit)
continue;
/* Give up on irreducible loops. */
if (bb->flags & BB_IRREDUCIBLE_LOOP)
{
precise = false;
break;
}
/* Check that inner loops are finite. */
for (class loop *l = bb->loop_father; l != loop && precise;
l = loop_outer (l))
if (!l->finite_p)
{
precise = false;
break;
}
/* Verify that there is no statement that may be terminate
execution in a way not visible to CFG. */
for (gimple_stmt_iterator bsi = gsi_start_bb (bb);
!gsi_end_p (bsi); gsi_next (&bsi))
if (stmt_can_terminate_bb_p (gsi_stmt (bsi)))
precise = false;
}
}
if (precise)
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
"Peeled all exits:"
" decreased number of iterations of loop %d by 1.\n",
loop->num);
adjust_loop_info_after_peeling (loop, 1, true);
}
else if (exit_count >= entry_count.apply_scale (9, 10))
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
"Peeled likely exits: likely decreased number "
"of iterations of loop %d by 1.\n", loop->num);
adjust_loop_info_after_peeling (loop, 1, false);
}
else if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
"Not decreased number"
" of iterations of loop %d; likely exits remains.\n",
loop->num);
changed = true;
}
+198 -37
View File
@@ -980,6 +980,75 @@ estimated_peeled_sequence_size (struct loop_size *size,
- size->eliminated_by_peeling), 1);
}
/* Update loop estimates after peeling LOOP by NPEEL.
If PRECISE is false only likely exists were duplicated and thus
do not update any estimates that are supposed to be always reliable. */
void
adjust_loop_info_after_peeling (class loop *loop, int npeel, bool precise)
{
if (loop->any_estimate)
{
/* Since peeling is mostly about loops where first few
iterations are special, it is not quite correct to
assume that the remaining iterations will behave
the same way. However we do not have better info
so update the esitmate, since it is likely better
than keeping it as it is.
Remove it if it looks wrong.
TODO: We likely want to special case the situation where
peeling is optimizing out exit edges and only update
estimates here. */
if (wi::leu_p (npeel, loop->nb_iterations_estimate))
loop->nb_iterations_estimate -= npeel;
else
loop->any_estimate = false;
}
if (loop->any_upper_bound && precise)
{
if (wi::leu_p (npeel, loop->nb_iterations_upper_bound))
loop->nb_iterations_upper_bound -= npeel;
else
{
/* Peeling maximal number of iterations or more
makes no sense and is a bug.
We should peel completely. */
gcc_unreachable ();
}
}
if (loop->any_likely_upper_bound)
{
if (wi::leu_p (npeel, loop->nb_iterations_likely_upper_bound))
loop->nb_iterations_likely_upper_bound -= npeel;
else
{
loop->any_estimate = true;
loop->nb_iterations_estimate = 0;
loop->nb_iterations_likely_upper_bound = 0;
}
}
if (loop->counters)
{
histogram_counters_minus_upper_bound (loop->counters, npeel);
/* TODO: Update loop estimate if possible. */
}
}
/* Return, in percent, probability that loop will end in first N iterations. */
static int
loop_histogram_percentage (class loop *loop, int n)
{
gcov_type sum = loop->counters->sum;
gcov_type psum = 0;
if (n > (int)loop->counters->lin->length ())
return -1;
for (int i = 0; i < n; i++)
psum += (*loop->counters->lin)[i];
return RDIV (psum * 100, sum);
}
/* If the loop is expected to iterate N times and is
small enough, duplicate the loop body N+1 times before
the loop itself. This way the hot path will never
@@ -989,7 +1058,7 @@ estimated_peeled_sequence_size (struct loop_size *size,
static bool
try_peel_loop (class loop *loop,
edge exit, tree niter, bool may_be_zero,
HOST_WIDE_INT maxiter)
HOST_WIDE_INT maxiter, dump_user_location_t locus)
{
HOST_WIDE_INT npeel;
struct loop_size size;
@@ -1037,6 +1106,78 @@ try_peel_loop (class loop *loop,
npeel = estimated_loop_iterations_int (loop);
if (npeel < 0)
npeel = likely_max_loop_iterations_int (loop);
auto_vec<int> good_peels;
auto_vec<int> prcnt;
prcnt.safe_push (0);
bool histogram_peeling = flag_use_histogram_in_peeling && (loop->counters != NULL);
if (!loop->counters && loop->header->count.reliable_p ()
&& loop->header->count.nonzero_p ()
&& dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION | MSG_PRIORITY_USER_FACING, locus,
"missing loop histogram for loop %d.\n", loop->num);
HOST_WIDE_INT nonhistogram_npeel = npeel + 1;
if (histogram_peeling && loop->counters->sum != 0
&& loop->counters->lin->length ())
{
gcov_type sum = loop->counters->sum;
gcov_type rest = sum;
gcov_type psum = 0;
gcov_type osum = 0;
int good_percentage = param_profile_histogram_peel_prcnt;
int good_overall_percentage = param_profile_histogram_peel_overall_prcnt;
for (int i = 0; i < (int)loop->counters->lin->length (); i++)
{
gcov_type e = (*(loop->counters->lin))[i];
psum += e;
osum += e;
if ((maxiter >= 0 && maxiter <= i)
|| (i >= param_max_peel_times - 1)
|| rest == 0)
break;
// iteration has enough cumulated in partial sum and itself has at
// least 1 percent or we have complete peeling
if (e && ((100 * psum) / sum >= good_percentage || psum == rest)
&& ((100 * osum) / sum) >= good_overall_percentage)
{
prcnt.safe_push (prcnt.last () + (100 * psum) / sum);
good_peels.safe_push (i);
good_percentage = 0;
rest -= psum;
psum = 0;
}
good_percentage = MIN (100, good_percentage + param_profile_histogram_peel_prcnt);
}
if (!good_peels.is_empty ())
{
npeel = good_peels.pop ();
// we do not pop() prcnt because we want it to be last for current
// iteration
}
else
{
if (dump_file)
fprintf (dump_file,
"Not peeling: found no good candidates in the histogram\n");
return false;
}
}
else
{
if (!flag_peel_loops_without_histogram)
{
if (dump_file)
fprintf (dump_file, "Not peeling: histogram is not available\n");
return false;
}
if (npeel >= 0 && maxiter >= 0 && maxiter <= npeel)
{
if (dump_file)
fprintf (dump_file, "Not peeling: upper bound is known so can "
"unroll completely\n");
return false;
}
}
if (npeel < 0)
{
if (dump_file)
@@ -1044,17 +1185,11 @@ try_peel_loop (class loop *loop,
"estimated\n");
return false;
}
if (maxiter >= 0 && maxiter <= npeel)
{
if (dump_file)
fprintf (dump_file, "Not peeling: upper bound is known so can "
"unroll completely\n");
return false;
}
/* We want to peel estimated number of iterations + 1 (so we never
enter the loop on quick path). Check against PARAM_MAX_PEEL_TIMES
and be sure to avoid overflows. */
if (npeel > param_max_peel_times - 1)
{
if (dump_file)
@@ -1067,6 +1202,19 @@ try_peel_loop (class loop *loop,
/* Check peeled loops size. */
tree_estimate_loop_size (loop, exit, NULL, &size,
param_max_peeled_insns);
// we want to peel the max_peeled_insns
// if we peel percentage corresponding the whole linear section
while (!good_peels.is_empty ()
&& ((int) estimated_peeled_sequence_size (&size, (int) npeel)
* MIN (100, param_profile_histogram_size_lin
* param_profile_histogram_peel_prcnt)
> param_max_peeled_insns * (prcnt.last ())))
{
prcnt.pop ();
npeel = good_peels.pop ();
++npeel;
}
if ((peeled_size = estimated_peeled_sequence_size (&size, (int) npeel))
> param_max_peeled_insns)
{
@@ -1104,36 +1252,49 @@ try_peel_loop (class loop *loop,
return false;
}
free_original_copy_tables ();
if (dump_file && (dump_flags & TDF_DETAILS))
if (dump_enabled_p ())
{
fprintf (dump_file, "Peeled loop %d, %i times.\n",
loop->num, (int) npeel);
}
if (loop->any_estimate)
{
if (wi::ltu_p (npeel, loop->nb_iterations_estimate))
loop->nb_iterations_estimate -= npeel;
else
loop->nb_iterations_estimate = 0;
}
if (loop->any_upper_bound)
{
if (wi::ltu_p (npeel, loop->nb_iterations_upper_bound))
loop->nb_iterations_upper_bound -= npeel;
else
loop->nb_iterations_upper_bound = 0;
}
if (loop->any_likely_upper_bound)
{
if (wi::ltu_p (npeel, loop->nb_iterations_likely_upper_bound))
loop->nb_iterations_likely_upper_bound -= npeel;
else
bool close = false;
if (histogram_peeling)
{
loop->any_estimate = true;
loop->nb_iterations_estimate = 0;
loop->nb_iterations_likely_upper_bound = 0;
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, locus,
"peeled loop %d, %i times with histogram, %i%% of executions",
loop->num, (int)npeel, loop_histogram_percentage (loop, npeel));
if (nonhistogram_npeel != npeel)
{
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
" (without histogram would try to peel %i times",
(int)nonhistogram_npeel);
int p = loop_histogram_percentage (loop, nonhistogram_npeel);
if (p >= 0)
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
", %i%% of executions", p);
close = true;
}
}
else
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, locus,
"peeled loop %d, %i times", loop->num, (int)npeel);
if (profile_info && loop->header->count.ipa ().initialized_p ())
{
if (!close)
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, " (");
else
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, "; ");
close = true;
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
"header execution count %wd",
(HOST_WIDE_INT)loop->header->count.ipa ().to_gcov_type ());
}
if (close)
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, ")\n");
else
dump_printf (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING, "\n");
}
adjust_loop_info_after_peeling (loop, npeel, true);
profile_count entry_count = profile_count::zero ();
edge e;
@@ -1151,6 +1312,7 @@ try_peel_loop (class loop *loop,
bitmap_set_bit (peeled_loops, loop->num);
return true;
}
/* Adds a canonical induction variable to LOOP if suitable.
CREATE_IV is true if we may create a new iv. UL determines
which loops we are allowed to completely unroll. If TRY_EVAL is true, we try
@@ -1273,9 +1435,8 @@ canonicalize_loop_induction_variables (class loop *loop,
if (iv_niter)
create_canonical_iv (loop, exit, iv_niter);
}
if (ul == UL_ALL)
modified |= try_peel_loop (loop, exit, niter, may_be_zero, maxiter);
modified |= try_peel_loop (loop, exit, niter, may_be_zero, maxiter, locus);
return modified;
}
@@ -1600,7 +1761,7 @@ pass_complete_unroll::execute (function *fun)
re-peeling the same loop multiple times. */
if (flag_peel_loops)
peeled_loops = BITMAP_ALLOC (NULL);
unsigned int val = tree_unroll_loops_completely (flag_cunroll_grow_size,
unsigned int val = tree_unroll_loops_completely (flag_cunroll_grow_size,
true);
if (peeled_loops)
{
+24 -5
View File
@@ -67,7 +67,9 @@ create_iv (tree base, tree step, tree var, class loop *loop,
gimple_seq stmts;
tree vb, va;
enum tree_code incr_op = PLUS_EXPR;
edge pe = loop_preheader_edge (loop);
edge pe = NULL;
edge e;
edge_iterator ei;
if (var != NULL_TREE)
{
@@ -122,7 +124,11 @@ create_iv (tree base, tree step, tree var, class loop *loop,
loop (i.e. the step should be loop invariant). */
step = force_gimple_operand (step, &stmts, true, NULL_TREE);
if (stmts)
gsi_insert_seq_on_edge_immediate (pe, stmts);
{
if (!pe)
pe = loop_preheader_edge (loop);
gsi_insert_seq_on_edge_immediate (pe, stmts);
}
stmt = gimple_build_assign (va, incr_op, vb, step);
/* Prevent the increment from inheriting a bogus location if it is not put
@@ -151,11 +157,24 @@ create_iv (tree base, tree step, tree var, class loop *loop,
initial = force_gimple_operand (base, &stmts, true, var);
if (stmts)
gsi_insert_seq_on_edge_immediate (pe, stmts);
{
if (!pe)
pe = loop_preheader_edge (loop);
gsi_insert_seq_on_edge_immediate (pe, stmts);
}
phi = create_phi_node (vb, loop->header);
add_phi_arg (phi, initial, loop_preheader_edge (loop), UNKNOWN_LOCATION);
add_phi_arg (phi, va, loop_latch_edge (loop), UNKNOWN_LOCATION);
FOR_EACH_EDGE (e, ei, loop->header->preds)
{
if (e == loop_latch_edge (loop))
{
add_phi_arg (phi, va, loop_latch_edge (loop), UNKNOWN_LOCATION);
}
else
{
add_phi_arg (phi, initial, e, UNKNOWN_LOCATION);
}
}
}
/* Return the innermost superloop LOOP of USE_LOOP that is a superloop of
+20
View File
@@ -2941,6 +2941,17 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
/* Prolog iterates at most bound_prolog times, latch iterates at
most bound_prolog - 1 times. */
record_niter_bound (prolog, bound_prolog - 1, false, true);
/* TODO: Update counters or at least release vectors.
We do not produce prologs on modern x86 targets we test on
so this is not important for proof of concept. */
if (prolog->counters)
{
va_heap::release (prolog->counters->lin);
va_heap::release (prolog->counters->exp);
if (prolog->counters->mod)
va_heap::release (prolog->counters->mod);
prolog->counters = NULL;
}
delete_update_ssa ();
adjust_vec_debug_stmts ();
scev_reset ();
@@ -3077,6 +3088,15 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
/* -1 to convert loop iterations to latch iterations. */
record_niter_bound (epilog, bound - 1, false, true);
}
/* TODO: Compute epilog histograms or at least release the counters. */
if (epilog->counters)
{
va_heap::release (epilog->counters->lin);
va_heap::release (epilog->counters->exp);
if (epilog->counters->mod)
va_heap::release (epilog->counters->mod);
epilog->counters = NULL;
}
delete_update_ssa ();
adjust_vec_debug_stmts ();
+48 -1
View File
@@ -2070,6 +2070,38 @@ vect_analyze_loop_costing (loop_vec_info loop_vinfo,
return 0;
}
bool use_estimate = true;
/* With histogram we can determine how often the vectorized loop will be
reached. */
if (loop->counters
/* TODO: For epilogue we can compute new histogram. */
&& !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
&& loop->counters->sum
&& min_profitable_estimate <= (int)loop->counters->lin->length ())
{
gcov_type psum = 0;
for (int i = 0; i < min_profitable_estimate
&& i < (int) loop->counters->lin->length ();
i++)
psum += (*(loop->counters->lin))[i];
/* TODO: This is less precise than needed. If histogram was also
storing number of executions of the loop header (which may
be different from loop->header->count if i.e. inlining happen).
We could compute expected nunber of execution of the vectorized
loop. */
if (psum == loop->counters->sum
|| !maybe_hot_count_p (cfun, loop->header->count.apply_scale
(loop->counters->sum - psum,
loop->counters->sum)))
{
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"not vectorized: according to loop histogram "
"the vectorized loop would be cold.\n");
return 0;
}
use_estimate = false;
}
HOST_WIDE_INT estimated_niter;
/* If we are vectorizing an epilogue then we know the maximum number of
@@ -2080,7 +2112,8 @@ vect_analyze_loop_costing (loop_vec_info loop_vinfo,
= vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) - 1;
else
{
estimated_niter = estimated_stmt_executions_int (loop);
if (use_estimate)
estimated_niter = estimated_stmt_executions_int (loop);
if (estimated_niter == -1)
estimated_niter = likely_max_stmt_executions_int (loop);
}
@@ -11056,6 +11089,11 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
bias_for_lowest += lowest_vf - min_first_active;
bias_for_assumed += assumed_vf - min_first_active;
}
if (loop->counters)
{
histogram_counters_div_upper_bound (loop->counters, lowest_vf);
histogram_counters_minus_upper_bound (loop->counters, 1);
}
/* In these calculations the "- 1" converts loop iteration counts
back to latch counts. */
if (loop->any_upper_bound)
@@ -11102,6 +11140,15 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
assumed_vf) - 1
: wi::udiv_floor (loop->nb_iterations_estimate + bias_for_assumed,
assumed_vf) - 1);
/* TODO: Fix profile after vectorization. */
if (loop->counters)
{
va_heap::release (loop->counters->lin);
va_heap::release (loop->counters->exp);
if (loop->counters->mod)
va_heap::release (loop->counters->mod);
loop->counters = NULL;
}
if (dump_enabled_p ())
{
+124 -6
View File
@@ -42,6 +42,10 @@ along with GCC; see the file COPYING3. If not see
#include "gimple-pretty-print.h"
#include "dumpfile.h"
#include "builtins.h"
#include "cfgloop.h"
#include "tree-ssa-loop-manip.h"
#include "tree-ssa-loop.h"
#include "tree-ssa-loop-niter.h"
/* In this file value profile based optimizations are placed. Currently the
following optimizations are implemented (for more detailed descriptions
@@ -117,6 +121,20 @@ gimple_alloc_histogram_value (struct function *fun ATTRIBUTE_UNUSED,
histogram_value hist = (histogram_value) xcalloc (1, sizeof (*hist));
hist->hvalue.value = value;
hist->hvalue.stmt = stmt;
hist->hvalue.lp = NULL;
hist->type = type;
return hist;
}
histogram_value
gimple_alloc_histogram_value_loop (struct function *fun ATTRIBUTE_UNUSED,
enum hist_type type, tree value,
class loop *lp)
{
histogram_value hist = (histogram_value) xcalloc (1, sizeof (*hist));
hist->hvalue.value = value;
hist->hvalue.stmt = NULL;
hist->hvalue.lp = lp;
hist->type = type;
return hist;
}
@@ -247,6 +265,38 @@ dump_histogram_value (FILE *dump_file, histogram_value hist)
(int64_t) hist->hvalue.counters[i]);
}
break;
case HIST_TYPE_HISTOGRAM:
if (hist->hvalue.counters)
{
for (int i = 0; i <= param_profile_histogram_size_lin; ++i)
{
fprintf (dump_file,
"Histogram counter histogram%" PRId64 ":%" PRId64 ".\n",
(int64_t) i, (int64_t) hist->hvalue.counters[i]);
}
int lin2_log = floor_log2 (param_profile_histogram_size_lin);
for (int64_t i = lin2_log;
i < param_profile_histogram_size_exp + lin2_log; ++i)
{
fprintf (
dump_file,
"Histogram counter histogram%" PRId64 ":%" PRId64 ".\n",
(int64_t) 1 << i,
(int64_t) hist->hvalue
.counters[(param_profile_histogram_size_lin - lin2_log) + i]);
}
for (int i = 0; i < param_profile_histogram_size_mod; ++i)
{
fprintf (dump_file,
"Histogram counter histogram%" PRId64 " mod:%" PRId64
".\n",
(int64_t) i,
(int64_t) hist->hvalue
.counters[i + param_profile_histogram_size_lin
+ param_profile_histogram_size_exp]);
}
}
break;
case HIST_TYPE_POW2:
if (hist->hvalue.counters)
@@ -364,6 +414,11 @@ stream_in_histogram_value (class lto_input_block *ib, gimple *stmt)
ncounters = new_val->hdata.intvl.steps + 2;
break;
case HIST_TYPE_HISTOGRAM:
ncounters = param_profile_histogram_size_lin
+ param_profile_histogram_size_exp
+ param_profile_histogram_size_mod;
break;
case HIST_TYPE_POW2:
case HIST_TYPE_AVERAGE:
ncounters = 2;
@@ -1799,11 +1854,13 @@ gimple_divmod_values_to_profile (gimple *stmt, histogram_values *values)
op0 = gimple_assign_rhs1 (stmt);
if (TREE_CODE (divisor) == SSA_NAME)
/* Check for the case where the divisor is the same value most
of the time. */
values->safe_push (gimple_alloc_histogram_value (cfun,
HIST_TYPE_TOPN_VALUES,
stmt, divisor));
{
/* Check for the case where the divisor is the same value most
of the time. */
values->safe_push (
gimple_alloc_histogram_value (cfun, HIST_TYPE_TOPN_VALUES, stmt,
divisor));
}
/* For mod, check whether it is not often a noop (or replaceable by
a few subtractions). */
@@ -1816,6 +1873,7 @@ gimple_divmod_values_to_profile (gimple *stmt, histogram_values *values)
values->safe_push (gimple_alloc_histogram_value (cfun,
HIST_TYPE_POW2,
stmt, divisor));
val = build2 (TRUNC_DIV_EXPR, type, op0, divisor);
hist = gimple_alloc_histogram_value (cfun, HIST_TYPE_INTERVAL,
stmt, val);
@@ -1889,6 +1947,60 @@ gimple_stringops_values_to_profile (gimple *gs, histogram_values *values)
stmt, dest));
}
static void
gimple_histogram_values_to_profile (function *fun, histogram_values *values)
{
if (!flag_profile_loops)
return;
for (auto loop : loops_list (fun, 0))
{
tree var;
gimple_stmt_iterator gsi;
edge exit_edge = NULL;
auto_vec<edge> exits = get_loop_exit_edges (loop);
bool complex_exit = false;
bool exit_found = false;
/* If loop has only exits that can not be instrumented,
we can not profile histograms. */
for (auto exit : exits)
if (exit->flags & EDGE_FAKE)
;
else if (exit->flags & EDGE_COMPLEX)
complex_exit = true;
else
{
if (!exit_found)
exit_edge = exit;
else
{
exit_edge = NULL;
break;
}
exit_found = true;
}
if (!exit_found)
continue;
/* If there is only one non-fake exit edge, see if we know number of
iterations. */
tree niter;
edge ex;
if (!complex_exit && exit_edge
&& (niter = find_loop_niter (loop, &ex))
&& TREE_CODE (niter) == INTEGER_CST)
continue;
gsi = gsi_start_bb (loop->latch);
create_iv (build_int_cst_type (get_gcov_type (), 0),
build_int_cst (get_gcov_type (), 1), NULL_TREE, loop, &gsi,
true, &var, NULL);
values->safe_push (gimple_alloc_histogram_value_loop (fun,
HIST_TYPE_HISTOGRAM,
var, loop));
}
}
/* Find values inside STMT for that we want to measure histograms and adds
them to list VALUES. */
@@ -1908,7 +2020,7 @@ gimple_find_values_to_profile (histogram_values *values)
unsigned i;
histogram_value hist = NULL;
values->create (0);
gimple_histogram_values_to_profile (cfun, values);
FOR_EACH_BB_FN (bb, cfun)
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
gimple_values_to_profile (gsi_stmt (gsi), values);
@@ -1945,6 +2057,12 @@ gimple_find_values_to_profile (histogram_values *values)
hist->n_counters = 1;
break;
case HIST_TYPE_HISTOGRAM:
hist->n_counters = param_profile_histogram_size_lin
+ param_profile_histogram_size_exp
+ param_profile_histogram_size_mod;
break;
default:
gcc_unreachable ();
}
+16 -11
View File
@@ -23,15 +23,16 @@ along with GCC; see the file COPYING3. If not see
/* Supported histogram types. */
enum hist_type
{
HIST_TYPE_INTERVAL, /* Measures histogram of values inside a specified
interval. */
HIST_TYPE_POW2, /* Histogram of power of 2 values. */
HIST_TYPE_TOPN_VALUES, /* Tries to identify the N most common values. */
HIST_TYPE_INDIR_CALL, /* Tries to identify the function that is (almost)
HIST_TYPE_INTERVAL, /* Measures histogram of values inside a specified
interval. */
HIST_TYPE_POW2, /* Histogram of power of 2 values. */
HIST_TYPE_TOPN_VALUES, /* Tries to identify the N most common values. */
HIST_TYPE_INDIR_CALL, /* Tries to identify the function that is (almost)
called in indirect call */
HIST_TYPE_AVERAGE, /* Compute average value (sum of all values). */
HIST_TYPE_IOR, /* Used to compute expected alignment. */
HIST_TYPE_AVERAGE, /* Compute average value (sum of all values). */
HIST_TYPE_IOR, /* Used to compute expected alignment. */
HIST_TYPE_TIME_PROFILE, /* Used for time profile */
HIST_TYPE_HISTOGRAM, /* used values histogram */
HIST_TYPE_MAX
};
@@ -39,7 +40,8 @@ enum hist_type
#define HIST_TYPE_FOR_COUNTER(COUNTER) \
((enum hist_type) ((COUNTER) - GCOV_FIRST_VALUE_COUNTER))
class loop;
/* The value to measure. */
struct histogram_value_t
{
@@ -47,6 +49,7 @@ struct histogram_value_t
{
tree value; /* The value to profile. */
gimple *stmt; /* Insn containing the value. */
class loop *lp;
gcov_type *counters; /* Pointer to first counter. */
struct histogram_value_t *next; /* Linked list pointer. */
} hvalue;
@@ -72,9 +75,9 @@ typedef vec<histogram_value> histogram_values;
extern void gimple_find_values_to_profile (histogram_values *);
extern bool gimple_value_profile_transformations (void);
histogram_value gimple_alloc_histogram_value (struct function *, enum hist_type,
gimple *stmt = NULL,
tree value = NULL_TREE);
histogram_value
gimple_alloc_histogram_value (struct function *, enum hist_type,
gimple *stmt = NULL, tree value = NULL);
histogram_value gimple_histogram_value (struct function *, gimple *);
histogram_value gimple_histogram_value_of_type (struct function *, gimple *,
enum hist_type);
@@ -99,6 +102,8 @@ extern void gimple_init_gcov_profiler (void);
extern void gimple_gen_edge_profiler (int, edge);
extern void gimple_gen_interval_profiler (histogram_value, unsigned);
extern void gimple_gen_pow2_profiler (histogram_value, unsigned);
extern void
gimple_gen_histogram_profiler (histogram_value, unsigned);
extern void gimple_gen_topn_values_profiler (histogram_value, unsigned);
extern void gimple_gen_ic_profiler (histogram_value, unsigned);
extern void gimple_gen_ic_func_profiler (void);
+3 -1
View File
@@ -906,7 +906,9 @@ LIBGCOV_PROFILER = _gcov_interval_profiler \
_gcov_ior_profiler \
_gcov_ior_profiler_atomic \
_gcov_indirect_call_profiler_v4 \
_gcov_time_profiler
_gcov_time_profiler \
_gcov_histogram_profiler \
_gcov_histogram_profiler_atomic
LIBGCOV_INTERFACE = _gcov_dump _gcov_fork \
_gcov_execl _gcov_execlp \
_gcov_execle _gcov_execv _gcov_execvp _gcov_execve _gcov_reset \
+97
View File
@@ -26,6 +26,103 @@ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
#include "libgcov.h"
#if !defined(inhibit_libc)
#ifndef floor_log2
/*
* If value is less then 8 we increment corresponding counter
* otherwise we take its logarithm and increment corresponding counter
*/
/* For convenience, define 0 -> word_size. */
static inline int clz_hwi(gcov_type_unsigned x) {
if (x == 0)
return sizeof(gcov_type_unsigned) * 8;
if (sizeof(gcov_type_unsigned) == sizeof(long))
return __builtin_clzl(x);
if (sizeof(gcov_type_unsigned) == sizeof(long long))
return __builtin_clzll(x);
else
return __builtin_clz(x);
}
static inline gcov_type_unsigned floor_log2(gcov_type_unsigned x) {
return sizeof(gcov_type_unsigned) * 8 - clz_hwi(x) - 1;
}
#endif
#ifdef L_gcov_histogram_profiler
/*
* If value is less then 8 we increment corresponding counter
* otherwise we take its logarithm and increment corresponding counter
*/
void __gcov_histogram_profiler(gcov_type *counters, gcov_type value,
gcov_type hist_sizes) {
gcc_assert(hist_sizes >= 0 && value >= 0);
gcov_type_unsigned hist_size = hist_sizes;
gcov_type_unsigned u_value = value;
// uncode section sizes
gcov_type_unsigned mask = (((gcov_type_unsigned)1) << 10) - 1;
gcov_type_unsigned lin_size = hist_size & mask;
gcov_type_unsigned exp_size = (hist_size >> 10) & mask;
gcov_type_unsigned mod_size = hist_size >> 20;
gcov_type_unsigned tot_size = exp_size + lin_size;
// add to the regular histogram
if (u_value < lin_size) {
counters[value]++;
} else {
gcov_type_unsigned pow2 = floor_log2(u_value);
gcov_type_unsigned lin_pow2 = floor_log2(lin_size - 1);
if (lin_size < tot_size && pow2 == lin_pow2) {
counters[lin_size]++;
} else {
if ((lin_pow2 - lin_size) + tot_size > pow2) {
counters[pow2 + (lin_size - lin_pow2) - 1]++;
} else {
counters[tot_size - 1]++;
}
}
}
// add to the modular histogram
counters[tot_size + (u_value % mod_size)]++;
}
#endif
#if defined(L_gcov_histogram_profiler_atomic) && GCOV_SUPPORTS_ATOMIC
void __gcov_histogram_profiler_atomic(gcov_type *counters, gcov_type value,
gcov_type hist_sizes) {
gcc_assert(hist_sizes >= 0 && value >= 0);
gcov_type_unsigned hist_size = hist_sizes;
gcov_type_unsigned u_value = value;
// uncode section sizes
gcov_type_unsigned mask = (((gcov_type_unsigned)1) << 10) - 1;
gcov_type_unsigned lin_size = hist_size & mask;
gcov_type_unsigned exp_size = (hist_size >> 10) & mask;
gcov_type_unsigned mod_size = hist_size >> 20;
gcov_type_unsigned tot_size = exp_size + lin_size;
// add to the regular histogram
if (u_value < lin_size) {
__atomic_fetch_add(&counters[value], 1, __ATOMIC_RELAXED);
} else {
gcov_type_unsigned pow2 = floor_log2(u_value);
gcov_type_unsigned lin_pow2 = floor_log2(lin_size - 1);
if ((lin_pow2 - lin_size) + tot_size > pow2) {
__atomic_fetch_add(&counters[pow2 + (lin_size - lin_pow2) - 1], 1,
__ATOMIC_RELAXED);
} else {
__atomic_fetch_add(&counters[tot_size - 1], 1, __ATOMIC_RELAXED);
}
}
// add to the modular histogram
__atomic_fetch_add(&counters[tot_size + (u_value % mod_size)], 1,
__ATOMIC_RELAXED);
}
#endif
#ifdef L_gcov_interval_profiler
/* If VALUE is in interval <START, START + STEPS - 1>, then increases the
corresponding counter in COUNTERS. If the VALUE is above or below
+2
View File
@@ -317,6 +317,8 @@ extern void __gcov_merge_ior (gcov_type *, unsigned) ATTRIBUTE_HIDDEN;
extern void __gcov_interval_profiler (gcov_type *, gcov_type, int, unsigned);
extern void __gcov_interval_profiler_atomic (gcov_type *, gcov_type, int,
unsigned);
extern void __gcov_histogram_profiler(gcov_type *, gcov_type, gcov_type);
extern void __gcov_histogram_profiler_atomic(gcov_type *, gcov_type, gcov_type);
extern void __gcov_pow2_profiler (gcov_type *, gcov_type);
extern void __gcov_pow2_profiler_atomic (gcov_type *, gcov_type);
extern void __gcov_topn_values_profiler (gcov_type *, gcov_type);