/*
* Copyright © 2012 Intel Corporation
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library. If not, see .
*/
#include
#include "llvm/Config/llvm-config.h"
#if LLVM_VERSION_MINOR <= 2
#include "llvm/Function.h"
#include "llvm/InstrTypes.h"
#include "llvm/Instructions.h"
#include "llvm/IntrinsicInst.h"
#include "llvm/Module.h"
#else
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#endif /* LLVM_VERSION_MINOR <= 2 */
#include "llvm/Pass.h"
#if LLVM_VERSION_MINOR <= 1
#include "llvm/Support/IRBuilder.h"
#elif LLVM_VERSION_MINOR == 2
#include "llvm/IRBuilder.h"
#else
#include "llvm/IR/IRBuilder.h"
#endif /* LLVM_VERSION_MINOR <= 1 */
#include "llvm/Support/raw_ostream.h"
#include "llvm/PassManager.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/llvm_gen_backend.hpp"
#include "sys/map.hpp"
using namespace llvm;
namespace gbe {
class CustomLoopUnroll : public LoopPass
{
public:
static char ID;
CustomLoopUnroll() :
LoopPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired();
AU.addPreserved();
AU.addRequiredID(LoopSimplifyID);
AU.addPreservedID(LoopSimplifyID);
AU.addRequiredID(LCSSAID);
AU.addPreservedID(LCSSAID);
AU.addRequired();
AU.addPreserved();
// FIXME: Loop unroll requires LCSSA. And LCSSA requires dom info.
// If loop unroll does not preserve dom info then LCSSA pass on next
// loop will receive invalid dom info.
// For now, recreate dom info, if loop is unrolled.
AU.addPreserved();
}
// Returns the value associated with the given metadata node name (for
// example, "llvm.loop.unroll.count"). If no such named metadata node
// exists, then nullptr is returned.
static const ConstantInt *GetUnrollMetadataValue(const Loop *L,
StringRef Name) {
MDNode *LoopID = L->getLoopID();
if (!LoopID) return nullptr;
// First operand should refer to the loop id itself.
assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
const MDNode *MD = dyn_cast(LoopID->getOperand(i));
if (!MD) continue;
const MDString *S = dyn_cast(MD->getOperand(0));
if (!S) continue;
if (Name.equals(S->getString())) {
assert(MD->getNumOperands() == 2 &&
"Unroll hint metadata should have two operands.");
return cast(MD->getOperand(1));
}
}
return nullptr;
}
void setUnrollID(Loop *L, bool enable) {
if (!enable && disabledLoops.find(L) != disabledLoops.end())
return;
LLVMContext &Context = L->getHeader()->getContext();
SmallVector forceUnroll;
forceUnroll.push_back(MDString::get(Context, "llvm.loop.unroll.enable"));
forceUnroll.push_back(ConstantInt::get(Type::getInt1Ty(Context), enable));
MDNode *forceUnrollNode = MDNode::get(Context, forceUnroll);
SmallVector Vals;
Vals.push_back(NULL);
Vals.push_back(forceUnrollNode);
MDNode *NewLoopID = MDNode::get(Context, Vals);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
L->setLoopID(NewLoopID);
if (!enable)
disabledLoops.insert(L);
}
static bool hasPrivateLoadStore(Loop *L) {
const std::vector subLoops = L->getSubLoops();
std::set subBlocks, blocks;
for(auto l : subLoops)
for(auto bb : l->getBlocks())
subBlocks.insert(bb);
for(auto bb : L->getBlocks())
if (subBlocks.find(bb) == subBlocks.end())
blocks.insert(bb);
for(auto bb : blocks) {
for (BasicBlock::iterator inst = bb->begin(), instE = bb->end(); inst != instE; ++inst) {
unsigned addrSpace = 0;
if (isa(*inst)) {
LoadInst *ld = cast(&*inst);
addrSpace = ld->getPointerAddressSpace();
}
else if (isa(*inst)) {
StoreInst *st = cast(&*inst);
addrSpace = st->getPointerAddressSpace();
}
if (addrSpace == 0)
return true;
}
}
return false;
}
// If one loop has very large self trip count
// we don't want to unroll it.
// self trip count means trip count divide by the parent's trip count. for example
// for (int i = 0; i < 16; i++) {
// for (int j = 0; j < 4; j++) {
// for (int k = 0; k < 2; k++) {
// ...
// }
// ...
// }
// The inner loops j and k could be unrolled, but the loop i will not be unrolled.
// The return value true means the L could be unrolled, otherwise, it could not
// be unrolled.
bool handleParentLoops(Loop *L, LPPassManager &LPM) {
Loop *currL = L;
ScalarEvolution *SE = &getAnalysis();
BasicBlock *latchBlock = currL->getLoopLatch();
unsigned currTripCount = 0;
bool shouldUnroll = true;
if (latchBlock)
currTripCount = SE->getSmallConstantTripCount(L, latchBlock);
while(currL) {
Loop *parentL = currL->getParentLoop();
unsigned parentTripCount = 0;
if (parentL) {
BasicBlock *parentLatchBlock = parentL->getLoopLatch();
if (parentLatchBlock)
parentTripCount = SE->getSmallConstantTripCount(parentL, parentLatchBlock);
}
if ((parentTripCount != 0 && currTripCount / parentTripCount > 16) ||
(currTripCount > 32)) {
if (currL == L)
shouldUnroll = false;
setUnrollID(currL, false);
if (currL != L)
LPM.deleteLoopFromQueue(currL);
}
currL = parentL;
currTripCount = parentTripCount;
}
return shouldUnroll;
}
// Analyze the outermost BBs of this loop, if there are
// some private load or store, we change it's loop meta data
// to indicate more aggresive unrolling on it.
virtual bool runOnLoop(Loop *L, LPPassManager &LPM) {
const ConstantInt *Enable = GetUnrollMetadataValue(L, "llvm.loop.unroll.enable");
if (Enable)
return false;
const ConstantInt *Count = GetUnrollMetadataValue(L, "llvm.loop.unroll.count");
if (Count)
return false;
if (!handleParentLoops(L, LPM))
return false;
if (!hasPrivateLoadStore(L))
return false;
setUnrollID(L, true);
return true;
}
virtual const char *getPassName() const {
return "SPIR backend: custom loop unrolling pass";
}
private:
std::set disabledLoops;
};
char CustomLoopUnroll::ID = 0;
LoopPass *createCustomLoopUnrollPass() {
return new CustomLoopUnroll();
}
} // end namespace