diff options
Diffstat (limited to 'gnu/llvm/lib/ProfileData')
| -rw-r--r-- | gnu/llvm/lib/ProfileData/CMakeLists.txt | 6 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/Coverage/CMakeLists.txt | 11 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp | 580 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp | 709 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp | 183 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/Coverage/LLVMBuild.txt | 23 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/InstrProf.cpp | 471 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/InstrProfReader.cpp | 289 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/InstrProfWriter.cpp | 223 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/LLVMBuild.txt | 5 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp | 116 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/SampleProf.cpp | 24 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/SampleProfReader.cpp | 97 | ||||
| -rw-r--r-- | gnu/llvm/lib/ProfileData/SampleProfWriter.cpp | 60 |
14 files changed, 2431 insertions, 366 deletions
diff --git a/gnu/llvm/lib/ProfileData/CMakeLists.txt b/gnu/llvm/lib/ProfileData/CMakeLists.txt index 22cca4b44df..cd65762ae6a 100644 --- a/gnu/llvm/lib/ProfileData/CMakeLists.txt +++ b/gnu/llvm/lib/ProfileData/CMakeLists.txt @@ -2,9 +2,7 @@ add_llvm_library(LLVMProfileData InstrProf.cpp InstrProfReader.cpp InstrProfWriter.cpp - CoverageMapping.cpp - CoverageMappingWriter.cpp - CoverageMappingReader.cpp + ProfileSummaryBuilder.cpp SampleProf.cpp SampleProfReader.cpp SampleProfWriter.cpp @@ -15,3 +13,5 @@ add_llvm_library(LLVMProfileData DEPENDS intrinsics_gen ) + +add_subdirectory(Coverage) diff --git a/gnu/llvm/lib/ProfileData/Coverage/CMakeLists.txt b/gnu/llvm/lib/ProfileData/Coverage/CMakeLists.txt new file mode 100644 index 00000000000..035b8fdb8b3 --- /dev/null +++ b/gnu/llvm/lib/ProfileData/Coverage/CMakeLists.txt @@ -0,0 +1,11 @@ +add_llvm_library(LLVMCoverage + CoverageMapping.cpp + CoverageMappingWriter.cpp + CoverageMappingReader.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/ProfileData/Coverage + + DEPENDS + intrinsics_gen + ) diff --git a/gnu/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/gnu/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp new file mode 100644 index 00000000000..fcd4e24bdfc --- /dev/null +++ b/gnu/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -0,0 +1,580 @@ +//=-- CoverageMapping.cpp - Code coverage mapping support ---------*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for clang's and llvm's instrumentation based +// code coverage. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/Coverage/CoverageMapping.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/ProfileData/Coverage/CoverageMappingReader.h" +#include "llvm/ProfileData/InstrProfReader.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace coverage; + +#define DEBUG_TYPE "coverage-mapping" + +Counter CounterExpressionBuilder::get(const CounterExpression &E) { + auto It = ExpressionIndices.find(E); + if (It != ExpressionIndices.end()) + return Counter::getExpression(It->second); + unsigned I = Expressions.size(); + Expressions.push_back(E); + ExpressionIndices[E] = I; + return Counter::getExpression(I); +} + +void CounterExpressionBuilder::extractTerms( + Counter C, int Sign, SmallVectorImpl<std::pair<unsigned, int>> &Terms) { + switch (C.getKind()) { + case Counter::Zero: + break; + case Counter::CounterValueReference: + Terms.push_back(std::make_pair(C.getCounterID(), Sign)); + break; + case Counter::Expression: + const auto &E = Expressions[C.getExpressionID()]; + extractTerms(E.LHS, Sign, Terms); + extractTerms(E.RHS, E.Kind == CounterExpression::Subtract ? -Sign : Sign, + Terms); + break; + } +} + +Counter CounterExpressionBuilder::simplify(Counter ExpressionTree) { + // Gather constant terms. + llvm::SmallVector<std::pair<unsigned, int>, 32> Terms; + extractTerms(ExpressionTree, +1, Terms); + + // If there are no terms, this is just a zero. The algorithm below assumes at + // least one term. + if (Terms.size() == 0) + return Counter::getZero(); + + // Group the terms by counter ID. + std::sort(Terms.begin(), Terms.end(), + [](const std::pair<unsigned, int> &LHS, + const std::pair<unsigned, int> &RHS) { + return LHS.first < RHS.first; + }); + + // Combine terms by counter ID to eliminate counters that sum to zero. + auto Prev = Terms.begin(); + for (auto I = Prev + 1, E = Terms.end(); I != E; ++I) { + if (I->first == Prev->first) { + Prev->second += I->second; + continue; + } + ++Prev; + *Prev = *I; + } + Terms.erase(++Prev, Terms.end()); + + Counter C; + // Create additions. We do this before subtractions to avoid constructs like + // ((0 - X) + Y), as opposed to (Y - X). + for (auto Term : Terms) { + if (Term.second <= 0) + continue; + for (int I = 0; I < Term.second; ++I) + if (C.isZero()) + C = Counter::getCounter(Term.first); + else + C = get(CounterExpression(CounterExpression::Add, C, + Counter::getCounter(Term.first))); + } + + // Create subtractions. + for (auto Term : Terms) { + if (Term.second >= 0) + continue; + for (int I = 0; I < -Term.second; ++I) + C = get(CounterExpression(CounterExpression::Subtract, C, + Counter::getCounter(Term.first))); + } + return C; +} + +Counter CounterExpressionBuilder::add(Counter LHS, Counter RHS) { + return simplify(get(CounterExpression(CounterExpression::Add, LHS, RHS))); +} + +Counter CounterExpressionBuilder::subtract(Counter LHS, Counter RHS) { + return simplify( + get(CounterExpression(CounterExpression::Subtract, LHS, RHS))); +} + +void CounterMappingContext::dump(const Counter &C, + llvm::raw_ostream &OS) const { + switch (C.getKind()) { + case Counter::Zero: + OS << '0'; + return; + case Counter::CounterValueReference: + OS << '#' << C.getCounterID(); + break; + case Counter::Expression: { + if (C.getExpressionID() >= Expressions.size()) + return; + const auto &E = Expressions[C.getExpressionID()]; + OS << '('; + dump(E.LHS, OS); + OS << (E.Kind == CounterExpression::Subtract ? " - " : " + "); + dump(E.RHS, OS); + OS << ')'; + break; + } + } + if (CounterValues.empty()) + return; + Expected<int64_t> Value = evaluate(C); + if (auto E = Value.takeError()) { + llvm::consumeError(std::move(E)); + return; + } + OS << '[' << *Value << ']'; +} + +Expected<int64_t> CounterMappingContext::evaluate(const Counter &C) const { + switch (C.getKind()) { + case Counter::Zero: + return 0; + case Counter::CounterValueReference: + if (C.getCounterID() >= CounterValues.size()) + return errorCodeToError(errc::argument_out_of_domain); + return CounterValues[C.getCounterID()]; + case Counter::Expression: { + if (C.getExpressionID() >= Expressions.size()) + return errorCodeToError(errc::argument_out_of_domain); + const auto &E = Expressions[C.getExpressionID()]; + Expected<int64_t> LHS = evaluate(E.LHS); + if (!LHS) + return LHS; + Expected<int64_t> RHS = evaluate(E.RHS); + if (!RHS) + return RHS; + return E.Kind == CounterExpression::Subtract ? *LHS - *RHS : *LHS + *RHS; + } + } + llvm_unreachable("Unhandled CounterKind"); +} + +void FunctionRecordIterator::skipOtherFiles() { + while (Current != Records.end() && !Filename.empty() && + Filename != Current->Filenames[0]) + ++Current; + if (Current == Records.end()) + *this = FunctionRecordIterator(); +} + +Expected<std::unique_ptr<CoverageMapping>> +CoverageMapping::load(CoverageMappingReader &CoverageReader, + IndexedInstrProfReader &ProfileReader) { + auto Coverage = std::unique_ptr<CoverageMapping>(new CoverageMapping()); + + std::vector<uint64_t> Counts; + for (const auto &Record : CoverageReader) { + CounterMappingContext Ctx(Record.Expressions); + + Counts.clear(); + if (Error E = ProfileReader.getFunctionCounts( + Record.FunctionName, Record.FunctionHash, Counts)) { + instrprof_error IPE = InstrProfError::take(std::move(E)); + if (IPE == instrprof_error::hash_mismatch) { + Coverage->MismatchedFunctionCount++; + continue; + } else if (IPE != instrprof_error::unknown_function) + return make_error<InstrProfError>(IPE); + Counts.assign(Record.MappingRegions.size(), 0); + } + Ctx.setCounts(Counts); + + assert(!Record.MappingRegions.empty() && "Function has no regions"); + + StringRef OrigFuncName = Record.FunctionName; + if (Record.Filenames.empty()) + OrigFuncName = getFuncNameWithoutPrefix(OrigFuncName); + else + OrigFuncName = + getFuncNameWithoutPrefix(OrigFuncName, Record.Filenames[0]); + FunctionRecord Function(OrigFuncName, Record.Filenames); + for (const auto &Region : Record.MappingRegions) { + Expected<int64_t> ExecutionCount = Ctx.evaluate(Region.Count); + if (auto E = ExecutionCount.takeError()) { + llvm::consumeError(std::move(E)); + break; + } + Function.pushRegion(Region, *ExecutionCount); + } + if (Function.CountedRegions.size() != Record.MappingRegions.size()) { + Coverage->MismatchedFunctionCount++; + continue; + } + + Coverage->Functions.push_back(std::move(Function)); + } + + return std::move(Coverage); +} + +Expected<std::unique_ptr<CoverageMapping>> +CoverageMapping::load(StringRef ObjectFilename, StringRef ProfileFilename, + StringRef Arch) { + auto CounterMappingBuff = MemoryBuffer::getFileOrSTDIN(ObjectFilename); + if (std::error_code EC = CounterMappingBuff.getError()) + return errorCodeToError(EC); + auto CoverageReaderOrErr = + BinaryCoverageReader::create(CounterMappingBuff.get(), Arch); + if (Error E = CoverageReaderOrErr.takeError()) + return std::move(E); + auto CoverageReader = std::move(CoverageReaderOrErr.get()); + auto ProfileReaderOrErr = IndexedInstrProfReader::create(ProfileFilename); + if (Error E = ProfileReaderOrErr.takeError()) + return std::move(E); + auto ProfileReader = std::move(ProfileReaderOrErr.get()); + return load(*CoverageReader, *ProfileReader); +} + +namespace { +/// \brief Distributes functions into instantiation sets. +/// +/// An instantiation set is a collection of functions that have the same source +/// code, ie, template functions specializations. +class FunctionInstantiationSetCollector { + typedef DenseMap<std::pair<unsigned, unsigned>, + std::vector<const FunctionRecord *>> MapT; + MapT InstantiatedFunctions; + +public: + void insert(const FunctionRecord &Function, unsigned FileID) { + auto I = Function.CountedRegions.begin(), E = Function.CountedRegions.end(); + while (I != E && I->FileID != FileID) + ++I; + assert(I != E && "function does not cover the given file"); + auto &Functions = InstantiatedFunctions[I->startLoc()]; + Functions.push_back(&Function); + } + + MapT::iterator begin() { return InstantiatedFunctions.begin(); } + + MapT::iterator end() { return InstantiatedFunctions.end(); } +}; + +class SegmentBuilder { + std::vector<CoverageSegment> &Segments; + SmallVector<const CountedRegion *, 8> ActiveRegions; + + SegmentBuilder(std::vector<CoverageSegment> &Segments) : Segments(Segments) {} + + /// Start a segment with no count specified. + void startSegment(unsigned Line, unsigned Col) { + DEBUG(dbgs() << "Top level segment at " << Line << ":" << Col << "\n"); + Segments.emplace_back(Line, Col, /*IsRegionEntry=*/false); + } + + /// Start a segment with the given Region's count. + void startSegment(unsigned Line, unsigned Col, bool IsRegionEntry, + const CountedRegion &Region) { + // Avoid creating empty regions. + if (!Segments.empty() && Segments.back().Line == Line && + Segments.back().Col == Col) + Segments.pop_back(); + DEBUG(dbgs() << "Segment at " << Line << ":" << Col); + // Set this region's count. + if (Region.Kind != coverage::CounterMappingRegion::SkippedRegion) { + DEBUG(dbgs() << " with count " << Region.ExecutionCount); + Segments.emplace_back(Line, Col, Region.ExecutionCount, IsRegionEntry); + } else + Segments.emplace_back(Line, Col, IsRegionEntry); + DEBUG(dbgs() << "\n"); + } + + /// Start a segment for the given region. + void startSegment(const CountedRegion &Region) { + startSegment(Region.LineStart, Region.ColumnStart, true, Region); + } + + /// Pop the top region off of the active stack, starting a new segment with + /// the containing Region's count. + void popRegion() { + const CountedRegion *Active = ActiveRegions.back(); + unsigned Line = Active->LineEnd, Col = Active->ColumnEnd; + ActiveRegions.pop_back(); + if (ActiveRegions.empty()) + startSegment(Line, Col); + else + startSegment(Line, Col, false, *ActiveRegions.back()); + } + + void buildSegmentsImpl(ArrayRef<CountedRegion> Regions) { + for (const auto &Region : Regions) { + // Pop any regions that end before this one starts. + while (!ActiveRegions.empty() && + ActiveRegions.back()->endLoc() <= Region.startLoc()) + popRegion(); + // Add this region to the stack. + ActiveRegions.push_back(&Region); + startSegment(Region); + } + // Pop any regions that are left in the stack. + while (!ActiveRegions.empty()) + popRegion(); + } + + /// Sort a nested sequence of regions from a single file. + static void sortNestedRegions(MutableArrayRef<CountedRegion> Regions) { + std::sort(Regions.begin(), Regions.end(), [](const CountedRegion &LHS, + const CountedRegion &RHS) { + if (LHS.startLoc() != RHS.startLoc()) + return LHS.startLoc() < RHS.startLoc(); + if (LHS.endLoc() != RHS.endLoc()) + // When LHS completely contains RHS, we sort LHS first. + return RHS.endLoc() < LHS.endLoc(); + // If LHS and RHS cover the same area, we need to sort them according + // to their kinds so that the most suitable region will become "active" + // in combineRegions(). Because we accumulate counter values only from + // regions of the same kind as the first region of the area, prefer + // CodeRegion to ExpansionRegion and ExpansionRegion to SkippedRegion. + static_assert(coverage::CounterMappingRegion::CodeRegion < + coverage::CounterMappingRegion::ExpansionRegion && + coverage::CounterMappingRegion::ExpansionRegion < + coverage::CounterMappingRegion::SkippedRegion, + "Unexpected order of region kind values"); + return LHS.Kind < RHS.Kind; + }); + } + + /// Combine counts of regions which cover the same area. + static ArrayRef<CountedRegion> + combineRegions(MutableArrayRef<CountedRegion> Regions) { + if (Regions.empty()) + return Regions; + auto Active = Regions.begin(); + auto End = Regions.end(); + for (auto I = Regions.begin() + 1; I != End; ++I) { + if (Active->startLoc() != I->startLoc() || + Active->endLoc() != I->endLoc()) { + // Shift to the next region. + ++Active; + if (Active != I) + *Active = *I; + continue; + } + // Merge duplicate region. + // If CodeRegions and ExpansionRegions cover the same area, it's probably + // a macro which is fully expanded to another macro. In that case, we need + // to accumulate counts only from CodeRegions, or else the area will be + // counted twice. + // On the other hand, a macro may have a nested macro in its body. If the + // outer macro is used several times, the ExpansionRegion for the nested + // macro will also be added several times. These ExpansionRegions cover + // the same source locations and have to be combined to reach the correct + // value for that area. + // We add counts of the regions of the same kind as the active region + // to handle the both situations. + if (I->Kind == Active->Kind) + Active->ExecutionCount += I->ExecutionCount; + } + return Regions.drop_back(std::distance(++Active, End)); + } + +public: + /// Build a list of CoverageSegments from a list of Regions. + static std::vector<CoverageSegment> + buildSegments(MutableArrayRef<CountedRegion> Regions) { + std::vector<CoverageSegment> Segments; + SegmentBuilder Builder(Segments); + + sortNestedRegions(Regions); + ArrayRef<CountedRegion> CombinedRegions = combineRegions(Regions); + + Builder.buildSegmentsImpl(CombinedRegions); + return Segments; + } +}; +} + +std::vector<StringRef> CoverageMapping::getUniqueSourceFiles() const { + std::vector<StringRef> Filenames; + for (const auto &Function : getCoveredFunctions()) + Filenames.insert(Filenames.end(), Function.Filenames.begin(), + Function.Filenames.end()); + std::sort(Filenames.begin(), Filenames.end()); + auto Last = std::unique(Filenames.begin(), Filenames.end()); + Filenames.erase(Last, Filenames.end()); + return Filenames; +} + +static SmallBitVector gatherFileIDs(StringRef SourceFile, + const FunctionRecord &Function) { + SmallBitVector FilenameEquivalence(Function.Filenames.size(), false); + for (unsigned I = 0, E = Function.Filenames.size(); I < E; ++I) + if (SourceFile == Function.Filenames[I]) + FilenameEquivalence[I] = true; + return FilenameEquivalence; +} + +/// Return the ID of the file where the definition of the function is located. +static Optional<unsigned> findMainViewFileID(const FunctionRecord &Function) { + SmallBitVector IsNotExpandedFile(Function.Filenames.size(), true); + for (const auto &CR : Function.CountedRegions) + if (CR.Kind == CounterMappingRegion::ExpansionRegion) + IsNotExpandedFile[CR.ExpandedFileID] = false; + int I = IsNotExpandedFile.find_first(); + if (I == -1) + return None; + return I; +} + +/// Check if SourceFile is the file that contains the definition of +/// the Function. Return the ID of the file in that case or None otherwise. +static Optional<unsigned> findMainViewFileID(StringRef SourceFile, + const FunctionRecord &Function) { + Optional<unsigned> I = findMainViewFileID(Function); + if (I && SourceFile == Function.Filenames[*I]) + return I; + return None; +} + +static bool isExpansion(const CountedRegion &R, unsigned FileID) { + return R.Kind == CounterMappingRegion::ExpansionRegion && R.FileID == FileID; +} + +CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const { + CoverageData FileCoverage(Filename); + std::vector<coverage::CountedRegion> Regions; + + for (const auto &Function : Functions) { + auto MainFileID = findMainViewFileID(Filename, Function); + auto FileIDs = gatherFileIDs(Filename, Function); + for (const auto &CR : Function.CountedRegions) + if (FileIDs.test(CR.FileID)) { + Regions.push_back(CR); + if (MainFileID && isExpansion(CR, *MainFileID)) + FileCoverage.Expansions.emplace_back(CR, Function); + } + } + + DEBUG(dbgs() << "Emitting segments for file: " << Filename << "\n"); + FileCoverage.Segments = SegmentBuilder::buildSegments(Regions); + + return FileCoverage; +} + +std::vector<const FunctionRecord *> +CoverageMapping::getInstantiations(StringRef Filename) const { + FunctionInstantiationSetCollector InstantiationSetCollector; + for (const auto &Function : Functions) { + auto MainFileID = findMainViewFileID(Filename, Function); + if (!MainFileID) + continue; + InstantiationSetCollector.insert(Function, *MainFileID); + } + + std::vector<const FunctionRecord *> Result; + for (const auto &InstantiationSet : InstantiationSetCollector) { + if (InstantiationSet.second.size() < 2) + continue; + Result.insert(Result.end(), InstantiationSet.second.begin(), + InstantiationSet.second.end()); + } + return Result; +} + +CoverageData +CoverageMapping::getCoverageForFunction(const FunctionRecord &Function) const { + auto MainFileID = findMainViewFileID(Function); + if (!MainFileID) + return CoverageData(); + + CoverageData FunctionCoverage(Function.Filenames[*MainFileID]); + std::vector<coverage::CountedRegion> Regions; + for (const auto &CR : Function.CountedRegions) + if (CR.FileID == *MainFileID) { + Regions.push_back(CR); + if (isExpansion(CR, *MainFileID)) + FunctionCoverage.Expansions.emplace_back(CR, Function); + } + + DEBUG(dbgs() << "Emitting segments for function: " << Function.Name << "\n"); + FunctionCoverage.Segments = SegmentBuilder::buildSegments(Regions); + + return FunctionCoverage; +} + +CoverageData CoverageMapping::getCoverageForExpansion( + const ExpansionRecord &Expansion) const { + CoverageData ExpansionCoverage( + Expansion.Function.Filenames[Expansion.FileID]); + std::vector<coverage::CountedRegion> Regions; + for (const auto &CR : Expansion.Function.CountedRegions) + if (CR.FileID == Expansion.FileID) { + Regions.push_back(CR); + if (isExpansion(CR, Expansion.FileID)) + ExpansionCoverage.Expansions.emplace_back(CR, Expansion.Function); + } + + DEBUG(dbgs() << "Emitting segments for expansion of file " << Expansion.FileID + << "\n"); + ExpansionCoverage.Segments = SegmentBuilder::buildSegments(Regions); + + return ExpansionCoverage; +} + +namespace { +std::string getCoverageMapErrString(coveragemap_error Err) { + switch (Err) { + case coveragemap_error::success: + return "Success"; + case coveragemap_error::eof: + return "End of File"; + case coveragemap_error::no_data_found: + return "No coverage data found"; + case coveragemap_error::unsupported_version: + return "Unsupported coverage format version"; + case coveragemap_error::truncated: + return "Truncated coverage data"; + case coveragemap_error::malformed: + return "Malformed coverage data"; + } + llvm_unreachable("A value of coveragemap_error has no message."); +} + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class CoverageMappingErrorCategoryType : public std::error_category { + const char *name() const LLVM_NOEXCEPT override { return "llvm.coveragemap"; } + std::string message(int IE) const override { + return getCoverageMapErrString(static_cast<coveragemap_error>(IE)); + } +}; +} // end anonymous namespace + +std::string CoverageMapError::message() const { + return getCoverageMapErrString(Err); +} + +static ManagedStatic<CoverageMappingErrorCategoryType> ErrorCategory; + +const std::error_category &llvm::coverage::coveragemap_category() { + return *ErrorCategory; +} + +char CoverageMapError::ID = 0; diff --git a/gnu/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/gnu/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp new file mode 100644 index 00000000000..1a4b4f59084 --- /dev/null +++ b/gnu/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -0,0 +1,709 @@ +//=-- CoverageMappingReader.cpp - Code coverage mapping reader ----*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for reading coverage mapping data for +// instrumentation based coverage. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/Coverage/CoverageMappingReader.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/Object/MachOUniversal.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/LEB128.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace coverage; +using namespace object; + +#define DEBUG_TYPE "coverage-mapping" + +void CoverageMappingIterator::increment() { + // Check if all the records were read or if an error occurred while reading + // the next record. + if (auto E = Reader->readNextRecord(Record)) { + handleAllErrors(std::move(E), [&](const CoverageMapError &CME) { + if (CME.get() == coveragemap_error::eof) + *this = CoverageMappingIterator(); + else + llvm_unreachable("Unexpected error in coverage mapping iterator"); + }); + } +} + +Error RawCoverageReader::readULEB128(uint64_t &Result) { + if (Data.size() < 1) + return make_error<CoverageMapError>(coveragemap_error::truncated); + unsigned N = 0; + Result = decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N); + if (N > Data.size()) + return make_error<CoverageMapError>(coveragemap_error::malformed); + Data = Data.substr(N); + return Error::success(); +} + +Error RawCoverageReader::readIntMax(uint64_t &Result, uint64_t MaxPlus1) { + if (auto Err = readULEB128(Result)) + return Err; + if (Result >= MaxPlus1) + return make_error<CoverageMapError>(coveragemap_error::malformed); + return Error::success(); +} + +Error RawCoverageReader::readSize(uint64_t &Result) { + if (auto Err = readULEB128(Result)) + return Err; + // Sanity check the number. + if (Result > Data.size()) + return make_error<CoverageMapError>(coveragemap_error::malformed); + return Error::success(); +} + +Error RawCoverageReader::readString(StringRef &Result) { + uint64_t Length; + if (auto Err = readSize(Length)) + return Err; + Result = Data.substr(0, Length); + Data = Data.substr(Length); + return Error::success(); +} + +Error RawCoverageFilenamesReader::read() { + uint64_t NumFilenames; + if (auto Err = readSize(NumFilenames)) + return Err; + for (size_t I = 0; I < NumFilenames; ++I) { + StringRef Filename; + if (auto Err = readString(Filename)) + return Err; + Filenames.push_back(Filename); + } + return Error::success(); +} + +Error RawCoverageMappingReader::decodeCounter(unsigned Value, Counter &C) { + auto Tag = Value & Counter::EncodingTagMask; + switch (Tag) { + case Counter::Zero: + C = Counter::getZero(); + return Error::success(); + case Counter::CounterValueReference: + C = Counter::getCounter(Value >> Counter::EncodingTagBits); + return Error::success(); + default: + break; + } + Tag -= Counter::Expression; + switch (Tag) { + case CounterExpression::Subtract: + case CounterExpression::Add: { + auto ID = Value >> Counter::EncodingTagBits; + if (ID >= Expressions.size()) + return make_error<CoverageMapError>(coveragemap_error::malformed); + Expressions[ID].Kind = CounterExpression::ExprKind(Tag); + C = Counter::getExpression(ID); + break; + } + default: + return make_error<CoverageMapError>(coveragemap_error::malformed); + } + return Error::success(); +} + +Error RawCoverageMappingReader::readCounter(Counter &C) { + uint64_t EncodedCounter; + if (auto Err = + readIntMax(EncodedCounter, std::numeric_limits<unsigned>::max())) + return Err; + if (auto Err = decodeCounter(EncodedCounter, C)) + return Err; + return Error::success(); +} + +static const unsigned EncodingExpansionRegionBit = 1 + << Counter::EncodingTagBits; + +/// \brief Read the sub-array of regions for the given inferred file id. +/// \param NumFileIDs the number of file ids that are defined for this +/// function. +Error RawCoverageMappingReader::readMappingRegionsSubArray( + std::vector<CounterMappingRegion> &MappingRegions, unsigned InferredFileID, + size_t NumFileIDs) { + uint64_t NumRegions; + if (auto Err = readSize(NumRegions)) + return Err; + unsigned LineStart = 0; + for (size_t I = 0; I < NumRegions; ++I) { + Counter C; + CounterMappingRegion::RegionKind Kind = CounterMappingRegion::CodeRegion; + + // Read the combined counter + region kind. + uint64_t EncodedCounterAndRegion; + if (auto Err = readIntMax(EncodedCounterAndRegion, + std::numeric_limits<unsigned>::max())) + return Err; + unsigned Tag = EncodedCounterAndRegion & Counter::EncodingTagMask; + uint64_t ExpandedFileID = 0; + if (Tag != Counter::Zero) { + if (auto Err = decodeCounter(EncodedCounterAndRegion, C)) + return Err; + } else { + // Is it an expansion region? + if (EncodedCounterAndRegion & EncodingExpansionRegionBit) { + Kind = CounterMappingRegion::ExpansionRegion; + ExpandedFileID = EncodedCounterAndRegion >> + Counter::EncodingCounterTagAndExpansionRegionTagBits; + if (ExpandedFileID >= NumFileIDs) + return make_error<CoverageMapError>(coveragemap_error::malformed); + } else { + switch (EncodedCounterAndRegion >> + Counter::EncodingCounterTagAndExpansionRegionTagBits) { + case CounterMappingRegion::CodeRegion: + // Don't do anything when we have a code region with a zero counter. + break; + case CounterMappingRegion::SkippedRegion: + Kind = CounterMappingRegion::SkippedRegion; + break; + default: + return make_error<CoverageMapError>(coveragemap_error::malformed); + } + } + } + + // Read the source range. + uint64_t LineStartDelta, ColumnStart, NumLines, ColumnEnd; + if (auto Err = + readIntMax(LineStartDelta, std::numeric_limits<unsigned>::max())) + return Err; + if (auto Err = readULEB128(ColumnStart)) + return Err; + if (ColumnStart > std::numeric_limits<unsigned>::max()) + return make_error<CoverageMapError>(coveragemap_error::malformed); + if (auto Err = readIntMax(NumLines, std::numeric_limits<unsigned>::max())) + return Err; + if (auto Err = readIntMax(ColumnEnd, std::numeric_limits<unsigned>::max())) + return Err; + LineStart += LineStartDelta; + // Adjust the column locations for the empty regions that are supposed to + // cover whole lines. Those regions should be encoded with the + // column range (1 -> std::numeric_limits<unsigned>::max()), but because + // the encoded std::numeric_limits<unsigned>::max() is several bytes long, + // we set the column range to (0 -> 0) to ensure that the column start and + // column end take up one byte each. + // The std::numeric_limits<unsigned>::max() is used to represent a column + // position at the end of the line without knowing the length of that line. + if (ColumnStart == 0 && ColumnEnd == 0) { + ColumnStart = 1; + ColumnEnd = std::numeric_limits<unsigned>::max(); + } + + DEBUG({ + dbgs() << "Counter in file " << InferredFileID << " " << LineStart << ":" + << ColumnStart << " -> " << (LineStart + NumLines) << ":" + << ColumnEnd << ", "; + if (Kind == CounterMappingRegion::ExpansionRegion) + dbgs() << "Expands to file " << ExpandedFileID; + else + CounterMappingContext(Expressions).dump(C, dbgs()); + dbgs() << "\n"; + }); + + MappingRegions.push_back(CounterMappingRegion( + C, InferredFileID, ExpandedFileID, LineStart, ColumnStart, + LineStart + NumLines, ColumnEnd, Kind)); + } + return Error::success(); +} + +Error RawCoverageMappingReader::read() { + + // Read the virtual file mapping. + llvm::SmallVector<unsigned, 8> VirtualFileMapping; + uint64_t NumFileMappings; + if (auto Err = readSize(NumFileMappings)) + return Err; + for (size_t I = 0; I < NumFileMappings; ++I) { + uint64_t FilenameIndex; + if (auto Err = readIntMax(FilenameIndex, TranslationUnitFilenames.size())) + return Err; + VirtualFileMapping.push_back(FilenameIndex); + } + + // Construct the files using unique filenames and virtual file mapping. + for (auto I : VirtualFileMapping) { + Filenames.push_back(TranslationUnitFilenames[I]); + } + + // Read the expressions. + uint64_t NumExpressions; + if (auto Err = readSize(NumExpressions)) + return Err; + // Create an array of dummy expressions that get the proper counters + // when the expressions are read, and the proper kinds when the counters + // are decoded. + Expressions.resize( + NumExpressions, + CounterExpression(CounterExpression::Subtract, Counter(), Counter())); + for (size_t I = 0; I < NumExpressions; ++I) { + if (auto Err = readCounter(Expressions[I].LHS)) + return Err; + if (auto Err = readCounter(Expressions[I].RHS)) + return Err; + } + + // Read the mapping regions sub-arrays. + for (unsigned InferredFileID = 0, S = VirtualFileMapping.size(); + InferredFileID < S; ++InferredFileID) { + if (auto Err = readMappingRegionsSubArray(MappingRegions, InferredFileID, + VirtualFileMapping.size())) + return Err; + } + + // Set the counters for the expansion regions. + // i.e. Counter of expansion region = counter of the first region + // from the expanded file. + // Perform multiple passes to correctly propagate the counters through + // all the nested expansion regions. + SmallVector<CounterMappingRegion *, 8> FileIDExpansionRegionMapping; + FileIDExpansionRegionMapping.resize(VirtualFileMapping.size(), nullptr); + for (unsigned Pass = 1, S = VirtualFileMapping.size(); Pass < S; ++Pass) { + for (auto &R : MappingRegions) { + if (R.Kind != CounterMappingRegion::ExpansionRegion) + continue; + assert(!FileIDExpansionRegionMapping[R.ExpandedFileID]); + FileIDExpansionRegionMapping[R.ExpandedFileID] = &R; + } + for (auto &R : MappingRegions) { + if (FileIDExpansionRegionMapping[R.FileID]) { + FileIDExpansionRegionMapping[R.FileID]->Count = R.Count; + FileIDExpansionRegionMapping[R.FileID] = nullptr; + } + } + } + + return Error::success(); +} + +Expected<bool> RawCoverageMappingDummyChecker::isDummy() { + // A dummy coverage mapping data consists of just one region with zero count. + uint64_t NumFileMappings; + if (Error Err = readSize(NumFileMappings)) + return std::move(Err); + if (NumFileMappings != 1) + return false; + // We don't expect any specific value for the filename index, just skip it. + uint64_t FilenameIndex; + if (Error Err = + readIntMax(FilenameIndex, std::numeric_limits<unsigned>::max())) + return std::move(Err); + uint64_t NumExpressions; + if (Error Err = readSize(NumExpressions)) + return std::move(Err); + if (NumExpressions != 0) + return false; + uint64_t NumRegions; + if (Error Err = readSize(NumRegions)) + return std::move(Err); + if (NumRegions != 1) + return false; + uint64_t EncodedCounterAndRegion; + if (Error Err = readIntMax(EncodedCounterAndRegion, + std::numeric_limits<unsigned>::max())) + return std::move(Err); + unsigned Tag = EncodedCounterAndRegion & Counter::EncodingTagMask; + return Tag == Counter::Zero; +} + +Error InstrProfSymtab::create(SectionRef &Section) { + if (auto EC = Section.getContents(Data)) + return errorCodeToError(EC); + Address = Section.getAddress(); + return Error::success(); +} + +StringRef InstrProfSymtab::getFuncName(uint64_t Pointer, size_t Size) { + if (Pointer < Address) + return StringRef(); + auto Offset = Pointer - Address; + if (Offset + Size > Data.size()) + return StringRef(); + return Data.substr(Pointer - Address, Size); +} + +// Check if the mapping data is a dummy, i.e. is emitted for an unused function. +static Expected<bool> isCoverageMappingDummy(uint64_t Hash, StringRef Mapping) { + // The hash value of dummy mapping records is always zero. + if (Hash) + return false; + return RawCoverageMappingDummyChecker(Mapping).isDummy(); +} + +namespace { +struct CovMapFuncRecordReader { + // The interface to read coverage mapping function records for a module. + // + // \p Buf points to the buffer containing the \c CovHeader of the coverage + // mapping data associated with the module. + // + // Returns a pointer to the next \c CovHeader if it exists, or a pointer + // greater than \p End if not. + virtual Expected<const char *> readFunctionRecords(const char *Buf, + const char *End) = 0; + virtual ~CovMapFuncRecordReader() {} + template <class IntPtrT, support::endianness Endian> + static Expected<std::unique_ptr<CovMapFuncRecordReader>> + get(coverage::CovMapVersion Version, InstrProfSymtab &P, + std::vector<BinaryCoverageReader::ProfileMappingRecord> &R, + std::vector<StringRef> &F); +}; + +// A class for reading coverage mapping function records for a module. +template <coverage::CovMapVersion Version, class IntPtrT, + support::endianness Endian> +class VersionedCovMapFuncRecordReader : public CovMapFuncRecordReader { + typedef typename coverage::CovMapTraits< + Version, IntPtrT>::CovMapFuncRecordType FuncRecordType; + typedef typename coverage::CovMapTraits<Version, IntPtrT>::NameRefType + NameRefType; + + // Maps function's name references to the indexes of their records + // in \c Records. + llvm::DenseMap<NameRefType, size_t> FunctionRecords; + InstrProfSymtab &ProfileNames; + std::vector<StringRef> &Filenames; + std::vector<BinaryCoverageReader::ProfileMappingRecord> &Records; + + // Add the record to the collection if we don't already have a record that + // points to the same function name. This is useful to ignore the redundant + // records for the functions with ODR linkage. + // In addition, prefer records with real coverage mapping data to dummy + // records, which were emitted for inline functions which were seen but + // not used in the corresponding translation unit. + Error insertFunctionRecordIfNeeded(const FuncRecordType *CFR, + StringRef Mapping, size_t FilenamesBegin) { + uint64_t FuncHash = CFR->template getFuncHash<Endian>(); + NameRefType NameRef = CFR->template getFuncNameRef<Endian>(); + auto InsertResult = + FunctionRecords.insert(std::make_pair(NameRef, Records.size())); + if (InsertResult.second) { + StringRef FuncName; + if (Error Err = CFR->template getFuncName<Endian>(ProfileNames, FuncName)) + return Err; + Records.emplace_back(Version, FuncName, FuncHash, Mapping, FilenamesBegin, + Filenames.size() - FilenamesBegin); + return Error::success(); + } + // Update the existing record if it's a dummy and the new record is real. + size_t OldRecordIndex = InsertResult.first->second; + BinaryCoverageReader::ProfileMappingRecord &OldRecord = + Records[OldRecordIndex]; + Expected<bool> OldIsDummyExpected = isCoverageMappingDummy( + OldRecord.FunctionHash, OldRecord.CoverageMapping); + if (Error Err = OldIsDummyExpected.takeError()) + return Err; + if (!*OldIsDummyExpected) + return Error::success(); + Expected<bool> NewIsDummyExpected = + isCoverageMappingDummy(FuncHash, Mapping); + if (Error Err = NewIsDummyExpected.takeError()) + return Err; + if (*NewIsDummyExpected) + return Error::success(); + OldRecord.FunctionHash = FuncHash; + OldRecord.CoverageMapping = Mapping; + OldRecord.FilenamesBegin = FilenamesBegin; + OldRecord.FilenamesSize = Filenames.size() - FilenamesBegin; + return Error::success(); + } + +public: + VersionedCovMapFuncRecordReader( + InstrProfSymtab &P, + std::vector<BinaryCoverageReader::ProfileMappingRecord> &R, + std::vector<StringRef> &F) + : ProfileNames(P), Filenames(F), Records(R) {} + ~VersionedCovMapFuncRecordReader() override {} + + Expected<const char *> readFunctionRecords(const char *Buf, + const char *End) override { + using namespace support; + if (Buf + sizeof(CovMapHeader) > End) + return make_error<CoverageMapError>(coveragemap_error::malformed); + auto CovHeader = reinterpret_cast<const coverage::CovMapHeader *>(Buf); + uint32_t NRecords = CovHeader->getNRecords<Endian>(); + uint32_t FilenamesSize = CovHeader->getFilenamesSize<Endian>(); + uint32_t CoverageSize = CovHeader->getCoverageSize<Endian>(); + assert((CovMapVersion)CovHeader->getVersion<Endian>() == Version); + Buf = reinterpret_cast<const char *>(CovHeader + 1); + + // Skip past the function records, saving the start and end for later. + const char *FunBuf = Buf; + Buf += NRecords * sizeof(FuncRecordType); + const char *FunEnd = Buf; + + // Get the filenames. + if (Buf + FilenamesSize > End) + return make_error<CoverageMapError>(coveragemap_error::malformed); + size_t FilenamesBegin = Filenames.size(); + RawCoverageFilenamesReader Reader(StringRef(Buf, FilenamesSize), Filenames); + if (auto Err = Reader.read()) + return std::move(Err); + Buf += FilenamesSize; + + // We'll read the coverage mapping records in the loop below. + const char *CovBuf = Buf; + Buf += CoverageSize; + const char *CovEnd = Buf; + + if (Buf > End) + return make_error<CoverageMapError>(coveragemap_error::malformed); + // Each coverage map has an alignment of 8, so we need to adjust alignment + // before reading the next map. + Buf += alignmentAdjustment(Buf, 8); + + auto CFR = reinterpret_cast<const FuncRecordType *>(FunBuf); + while ((const char *)CFR < FunEnd) { + // Read the function information + uint32_t DataSize = CFR->template getDataSize<Endian>(); + + // Now use that to read the coverage data. + if (CovBuf + DataSize > CovEnd) + return make_error<CoverageMapError>(coveragemap_error::malformed); + auto Mapping = StringRef(CovBuf, DataSize); + CovBuf += DataSize; + + if (Error Err = + insertFunctionRecordIfNeeded(CFR, Mapping, FilenamesBegin)) + return std::move(Err); + CFR++; + } + return Buf; + } +}; +} // end anonymous namespace + +template <class IntPtrT, support::endianness Endian> +Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get( + coverage::CovMapVersion Version, InstrProfSymtab &P, + std::vector<BinaryCoverageReader::ProfileMappingRecord> &R, + std::vector<StringRef> &F) { + using namespace coverage; + switch (Version) { + case CovMapVersion::Version1: + return llvm::make_unique<VersionedCovMapFuncRecordReader< + CovMapVersion::Version1, IntPtrT, Endian>>(P, R, F); + case CovMapVersion::Version2: + // Decompress the name data. + if (Error E = P.create(P.getNameData())) + return std::move(E); + return llvm::make_unique<VersionedCovMapFuncRecordReader< + CovMapVersion::Version2, IntPtrT, Endian>>(P, R, F); + } + llvm_unreachable("Unsupported version"); +} + +template <typename T, support::endianness Endian> +static Error readCoverageMappingData( + InstrProfSymtab &ProfileNames, StringRef Data, + std::vector<BinaryCoverageReader::ProfileMappingRecord> &Records, + std::vector<StringRef> &Filenames) { + using namespace coverage; + // Read the records in the coverage data section. + auto CovHeader = + reinterpret_cast<const coverage::CovMapHeader *>(Data.data()); + CovMapVersion Version = (CovMapVersion)CovHeader->getVersion<Endian>(); + if (Version > coverage::CovMapVersion::CurrentVersion) + return make_error<CoverageMapError>(coveragemap_error::unsupported_version); + Expected<std::unique_ptr<CovMapFuncRecordReader>> ReaderExpected = + CovMapFuncRecordReader::get<T, Endian>(Version, ProfileNames, Records, + Filenames); + if (Error E = ReaderExpected.takeError()) + return E; + auto Reader = std::move(ReaderExpected.get()); + for (const char *Buf = Data.data(), *End = Buf + Data.size(); Buf < End;) { + auto NextHeaderOrErr = Reader->readFunctionRecords(Buf, End); + if (auto E = NextHeaderOrErr.takeError()) + return E; + Buf = NextHeaderOrErr.get(); + } + return Error::success(); +} +static const char *TestingFormatMagic = "llvmcovmtestdata"; + +static Error loadTestingFormat(StringRef Data, InstrProfSymtab &ProfileNames, + StringRef &CoverageMapping, + uint8_t &BytesInAddress, + support::endianness &Endian) { + BytesInAddress = 8; + Endian = support::endianness::little; + + Data = Data.substr(StringRef(TestingFormatMagic).size()); + if (Data.size() < 1) + return make_error<CoverageMapError>(coveragemap_error::truncated); + unsigned N = 0; + auto ProfileNamesSize = + decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N); + if (N > Data.size()) + return make_error<CoverageMapError>(coveragemap_error::malformed); + Data = Data.substr(N); + if (Data.size() < 1) + return make_error<CoverageMapError>(coveragemap_error::truncated); + N = 0; + uint64_t Address = + decodeULEB128(reinterpret_cast<const uint8_t *>(Data.data()), &N); + if (N > Data.size()) + return make_error<CoverageMapError>(coveragemap_error::malformed); + Data = Data.substr(N); + if (Data.size() < ProfileNamesSize) + return make_error<CoverageMapError>(coveragemap_error::malformed); + if (Error E = ProfileNames.create(Data.substr(0, ProfileNamesSize), Address)) + return E; + CoverageMapping = Data.substr(ProfileNamesSize); + // Skip the padding bytes because coverage map data has an alignment of 8. + if (CoverageMapping.size() < 1) + return make_error<CoverageMapError>(coveragemap_error::truncated); + size_t Pad = alignmentAdjustment(CoverageMapping.data(), 8); + if (CoverageMapping.size() < Pad) + return make_error<CoverageMapError>(coveragemap_error::malformed); + CoverageMapping = CoverageMapping.substr(Pad); + return Error::success(); +} + +static Expected<SectionRef> lookupSection(ObjectFile &OF, StringRef Name) { + StringRef FoundName; + for (const auto &Section : OF.sections()) { + if (auto EC = Section.getName(FoundName)) + return errorCodeToError(EC); + if (FoundName == Name) + return Section; + } + return make_error<CoverageMapError>(coveragemap_error::no_data_found); +} + +static Error loadBinaryFormat(MemoryBufferRef ObjectBuffer, + InstrProfSymtab &ProfileNames, + StringRef &CoverageMapping, + uint8_t &BytesInAddress, + support::endianness &Endian, StringRef Arch) { + auto BinOrErr = object::createBinary(ObjectBuffer); + if (!BinOrErr) + return BinOrErr.takeError(); + auto Bin = std::move(BinOrErr.get()); + std::unique_ptr<ObjectFile> OF; + if (auto *Universal = dyn_cast<object::MachOUniversalBinary>(Bin.get())) { + // If we have a universal binary, try to look up the object for the + // appropriate architecture. + auto ObjectFileOrErr = Universal->getObjectForArch(Arch); + if (!ObjectFileOrErr) + return ObjectFileOrErr.takeError(); + OF = std::move(ObjectFileOrErr.get()); + } else if (isa<object::ObjectFile>(Bin.get())) { + // For any other object file, upcast and take ownership. + OF.reset(cast<object::ObjectFile>(Bin.release())); + // If we've asked for a particular arch, make sure they match. + if (!Arch.empty() && OF->getArch() != Triple(Arch).getArch()) + return errorCodeToError(object_error::arch_not_found); + } else + // We can only handle object files. + return make_error<CoverageMapError>(coveragemap_error::malformed); + + // The coverage uses native pointer sizes for the object it's written in. + BytesInAddress = OF->getBytesInAddress(); + Endian = OF->isLittleEndian() ? support::endianness::little + : support::endianness::big; + + // Look for the sections that we are interested in. + auto NamesSection = lookupSection(*OF, getInstrProfNameSectionName(false)); + if (auto E = NamesSection.takeError()) + return E; + auto CoverageSection = + lookupSection(*OF, getInstrProfCoverageSectionName(false)); + if (auto E = CoverageSection.takeError()) + return E; + + // Get the contents of the given sections. + if (auto EC = CoverageSection->getContents(CoverageMapping)) + return errorCodeToError(EC); + if (Error E = ProfileNames.create(*NamesSection)) + return E; + + return Error::success(); +} + +Expected<std::unique_ptr<BinaryCoverageReader>> +BinaryCoverageReader::create(std::unique_ptr<MemoryBuffer> &ObjectBuffer, + StringRef Arch) { + std::unique_ptr<BinaryCoverageReader> Reader(new BinaryCoverageReader()); + + StringRef Coverage; + uint8_t BytesInAddress; + support::endianness Endian; + Error E; + consumeError(std::move(E)); + if (ObjectBuffer->getBuffer().startswith(TestingFormatMagic)) + // This is a special format used for testing. + E = loadTestingFormat(ObjectBuffer->getBuffer(), Reader->ProfileNames, + Coverage, BytesInAddress, Endian); + else + E = loadBinaryFormat(ObjectBuffer->getMemBufferRef(), Reader->ProfileNames, + Coverage, BytesInAddress, Endian, Arch); + if (E) + return std::move(E); + + if (BytesInAddress == 4 && Endian == support::endianness::little) + E = readCoverageMappingData<uint32_t, support::endianness::little>( + Reader->ProfileNames, Coverage, Reader->MappingRecords, + Reader->Filenames); + else if (BytesInAddress == 4 && Endian == support::endianness::big) + E = readCoverageMappingData<uint32_t, support::endianness::big>( + Reader->ProfileNames, Coverage, Reader->MappingRecords, + Reader->Filenames); + else if (BytesInAddress == 8 && Endian == support::endianness::little) + E = readCoverageMappingData<uint64_t, support::endianness::little>( + Reader->ProfileNames, Coverage, Reader->MappingRecords, + Reader->Filenames); + else if (BytesInAddress == 8 && Endian == support::endianness::big) + E = readCoverageMappingData<uint64_t, support::endianness::big>( + Reader->ProfileNames, Coverage, Reader->MappingRecords, + Reader->Filenames); + else + return make_error<CoverageMapError>(coveragemap_error::malformed); + if (E) + return std::move(E); + return std::move(Reader); +} + +Error BinaryCoverageReader::readNextRecord(CoverageMappingRecord &Record) { + if (CurrentRecord >= MappingRecords.size()) + return make_error<CoverageMapError>(coveragemap_error::eof); + + FunctionsFilenames.clear(); + Expressions.clear(); + MappingRegions.clear(); + auto &R = MappingRecords[CurrentRecord]; + RawCoverageMappingReader Reader( + R.CoverageMapping, + makeArrayRef(Filenames).slice(R.FilenamesBegin, R.FilenamesSize), + FunctionsFilenames, Expressions, MappingRegions); + if (auto Err = Reader.read()) + return Err; + + Record.FunctionName = R.FunctionName; + Record.FunctionHash = R.FunctionHash; + Record.Filenames = FunctionsFilenames; + Record.Expressions = Expressions; + Record.MappingRegions = MappingRegions; + + ++CurrentRecord; + return Error::success(); +} diff --git a/gnu/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/gnu/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp new file mode 100644 index 00000000000..8ff90d62cfd --- /dev/null +++ b/gnu/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp @@ -0,0 +1,183 @@ +//=-- CoverageMappingWriter.cpp - Code coverage mapping writer -------------=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for writing coverage mapping data for +// instrumentation based coverage. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" +#include "llvm/Support/LEB128.h" + +using namespace llvm; +using namespace coverage; + +void CoverageFilenamesSectionWriter::write(raw_ostream &OS) { + encodeULEB128(Filenames.size(), OS); + for (const auto &Filename : Filenames) { + encodeULEB128(Filename.size(), OS); + OS << Filename; + } +} + +namespace { +/// \brief Gather only the expressions that are used by the mapping +/// regions in this function. +class CounterExpressionsMinimizer { + ArrayRef<CounterExpression> Expressions; + llvm::SmallVector<CounterExpression, 16> UsedExpressions; + std::vector<unsigned> AdjustedExpressionIDs; + +public: + void mark(Counter C) { + if (!C.isExpression()) + return; + unsigned ID = C.getExpressionID(); + AdjustedExpressionIDs[ID] = 1; + mark(Expressions[ID].LHS); + mark(Expressions[ID].RHS); + } + + void gatherUsed(Counter C) { + if (!C.isExpression() || !AdjustedExpressionIDs[C.getExpressionID()]) + return; + AdjustedExpressionIDs[C.getExpressionID()] = UsedExpressions.size(); + const auto &E = Expressions[C.getExpressionID()]; + UsedExpressions.push_back(E); + gatherUsed(E.LHS); + gatherUsed(E.RHS); + } + + CounterExpressionsMinimizer(ArrayRef<CounterExpression> Expressions, + ArrayRef<CounterMappingRegion> MappingRegions) + : Expressions(Expressions) { + AdjustedExpressionIDs.resize(Expressions.size(), 0); + for (const auto &I : MappingRegions) + mark(I.Count); + for (const auto &I : MappingRegions) + gatherUsed(I.Count); + } + + ArrayRef<CounterExpression> getExpressions() const { return UsedExpressions; } + + /// \brief Adjust the given counter to correctly transition from the old + /// expression ids to the new expression ids. + Counter adjust(Counter C) const { + if (C.isExpression()) + C = Counter::getExpression(AdjustedExpressionIDs[C.getExpressionID()]); + return C; + } +}; +} + +/// \brief Encode the counter. +/// +/// The encoding uses the following format: +/// Low 2 bits - Tag: +/// Counter::Zero(0) - A Counter with kind Counter::Zero +/// Counter::CounterValueReference(1) - A counter with kind +/// Counter::CounterValueReference +/// Counter::Expression(2) + CounterExpression::Subtract(0) - +/// A counter with kind Counter::Expression and an expression +/// with kind CounterExpression::Subtract +/// Counter::Expression(2) + CounterExpression::Add(1) - +/// A counter with kind Counter::Expression and an expression +/// with kind CounterExpression::Add +/// Remaining bits - Counter/Expression ID. +static unsigned encodeCounter(ArrayRef<CounterExpression> Expressions, + Counter C) { + unsigned Tag = unsigned(C.getKind()); + if (C.isExpression()) + Tag += Expressions[C.getExpressionID()].Kind; + unsigned ID = C.getCounterID(); + assert(ID <= + (std::numeric_limits<unsigned>::max() >> Counter::EncodingTagBits)); + return Tag | (ID << Counter::EncodingTagBits); +} + +static void writeCounter(ArrayRef<CounterExpression> Expressions, Counter C, + raw_ostream &OS) { + encodeULEB128(encodeCounter(Expressions, C), OS); +} + +void CoverageMappingWriter::write(raw_ostream &OS) { + // Sort the regions in an ascending order by the file id and the starting + // location. + std::stable_sort(MappingRegions.begin(), MappingRegions.end()); + + // Write out the fileid -> filename mapping. + encodeULEB128(VirtualFileMapping.size(), OS); + for (const auto &FileID : VirtualFileMapping) + encodeULEB128(FileID, OS); + + // Write out the expressions. + CounterExpressionsMinimizer Minimizer(Expressions, MappingRegions); + auto MinExpressions = Minimizer.getExpressions(); + encodeULEB128(MinExpressions.size(), OS); + for (const auto &E : MinExpressions) { + writeCounter(MinExpressions, Minimizer.adjust(E.LHS), OS); + writeCounter(MinExpressions, Minimizer.adjust(E.RHS), OS); + } + + // Write out the mapping regions. + // Split the regions into subarrays where each region in a + // subarray has a fileID which is the index of that subarray. + unsigned PrevLineStart = 0; + unsigned CurrentFileID = ~0U; + for (auto I = MappingRegions.begin(), E = MappingRegions.end(); I != E; ++I) { + if (I->FileID != CurrentFileID) { + // Ensure that all file ids have at least one mapping region. + assert(I->FileID == (CurrentFileID + 1)); + // Find the number of regions with this file id. + unsigned RegionCount = 1; + for (auto J = I + 1; J != E && I->FileID == J->FileID; ++J) + ++RegionCount; + // Start a new region sub-array. + encodeULEB128(RegionCount, OS); + + CurrentFileID = I->FileID; + PrevLineStart = 0; + } + Counter Count = Minimizer.adjust(I->Count); + switch (I->Kind) { + case CounterMappingRegion::CodeRegion: + writeCounter(MinExpressions, Count, OS); + break; + case CounterMappingRegion::ExpansionRegion: { + assert(Count.isZero()); + assert(I->ExpandedFileID <= + (std::numeric_limits<unsigned>::max() >> + Counter::EncodingCounterTagAndExpansionRegionTagBits)); + // Mark an expansion region with a set bit that follows the counter tag, + // and pack the expanded file id into the remaining bits. + unsigned EncodedTagExpandedFileID = + (1 << Counter::EncodingTagBits) | + (I->ExpandedFileID + << Counter::EncodingCounterTagAndExpansionRegionTagBits); + encodeULEB128(EncodedTagExpandedFileID, OS); + break; + } + case CounterMappingRegion::SkippedRegion: + assert(Count.isZero()); + encodeULEB128(unsigned(I->Kind) + << Counter::EncodingCounterTagAndExpansionRegionTagBits, + OS); + break; + } + assert(I->LineStart >= PrevLineStart); + encodeULEB128(I->LineStart - PrevLineStart, OS); + encodeULEB128(I->ColumnStart, OS); + assert(I->LineEnd >= I->LineStart); + encodeULEB128(I->LineEnd - I->LineStart, OS); + encodeULEB128(I->ColumnEnd, OS); + PrevLineStart = I->LineStart; + } + // Ensure that all file ids have at least one mapping region. + assert(CurrentFileID == (VirtualFileMapping.size() - 1)); +} diff --git a/gnu/llvm/lib/ProfileData/Coverage/LLVMBuild.txt b/gnu/llvm/lib/ProfileData/Coverage/LLVMBuild.txt new file mode 100644 index 00000000000..fc8284b0ef3 --- /dev/null +++ b/gnu/llvm/lib/ProfileData/Coverage/LLVMBuild.txt @@ -0,0 +1,23 @@ +;===- ./lib/ProfileData/Coverage/LLVMBuild.txt -----------------*- Conf -*--===; +; +; The LLVM Compiler Infrastructure +; +; This file is distributed under the University of Illinois Open Source +; License. See LICENSE.TXT for details. +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = Coverage +parent = ProfileData +required_libraries = Core Object ProfileData Support + diff --git a/gnu/llvm/lib/ProfileData/InstrProf.cpp b/gnu/llvm/lib/ProfileData/InstrProf.cpp index d6777639abe..6962f82a5ef 100644 --- a/gnu/llvm/lib/ProfileData/InstrProf.cpp +++ b/gnu/llvm/lib/ProfileData/InstrProf.cpp @@ -17,55 +17,72 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/Compression.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" using namespace llvm; +static cl::opt<bool> StaticFuncFullModulePrefix( + "static-func-full-module-prefix", cl::init(false), + cl::desc("Use full module build paths in the profile counter names for " + "static functions.")); + namespace { +std::string getInstrProfErrString(instrprof_error Err) { + switch (Err) { + case instrprof_error::success: + return "Success"; + case instrprof_error::eof: + return "End of File"; + case instrprof_error::unrecognized_format: + return "Unrecognized instrumentation profile encoding format"; + case instrprof_error::bad_magic: + return "Invalid instrumentation profile data (bad magic)"; + case instrprof_error::bad_header: + return "Invalid instrumentation profile data (file header is corrupt)"; + case instrprof_error::unsupported_version: + return "Unsupported instrumentation profile format version"; + case instrprof_error::unsupported_hash_type: + return "Unsupported instrumentation profile hash type"; + case instrprof_error::too_large: + return "Too much profile data"; + case instrprof_error::truncated: + return "Truncated profile data"; + case instrprof_error::malformed: + return "Malformed instrumentation profile data"; + case instrprof_error::unknown_function: + return "No profile data available for function"; + case instrprof_error::hash_mismatch: + return "Function control flow change detected (hash mismatch)"; + case instrprof_error::count_mismatch: + return "Function basic block count change detected (counter mismatch)"; + case instrprof_error::counter_overflow: + return "Counter overflow"; + case instrprof_error::value_site_count_mismatch: + return "Function value site count change detected (counter mismatch)"; + case instrprof_error::compress_failed: + return "Failed to compress data (zlib)"; + case instrprof_error::uncompress_failed: + return "Failed to uncompress data (zlib)"; + } + llvm_unreachable("A value of instrprof_error has no message."); +} + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. class InstrProfErrorCategoryType : public std::error_category { const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; } std::string message(int IE) const override { - instrprof_error E = static_cast<instrprof_error>(IE); - switch (E) { - case instrprof_error::success: - return "Success"; - case instrprof_error::eof: - return "End of File"; - case instrprof_error::unrecognized_format: - return "Unrecognized instrumentation profile encoding format"; - case instrprof_error::bad_magic: - return "Invalid instrumentation profile data (bad magic)"; - case instrprof_error::bad_header: - return "Invalid instrumentation profile data (file header is corrupt)"; - case instrprof_error::unsupported_version: - return "Unsupported instrumentation profile format version"; - case instrprof_error::unsupported_hash_type: - return "Unsupported instrumentation profile hash type"; - case instrprof_error::too_large: - return "Too much profile data"; - case instrprof_error::truncated: - return "Truncated profile data"; - case instrprof_error::malformed: - return "Malformed instrumentation profile data"; - case instrprof_error::unknown_function: - return "No profile data available for function"; - case instrprof_error::hash_mismatch: - return "Function control flow change detected (hash mismatch)"; - case instrprof_error::count_mismatch: - return "Function basic block count change detected (counter mismatch)"; - case instrprof_error::counter_overflow: - return "Counter overflow"; - case instrprof_error::value_site_count_mismatch: - return "Function value site count change detected (counter mismatch)"; - } - llvm_unreachable("A value of instrprof_error has no message."); + return getInstrProfErrString(static_cast<instrprof_error>(IE)); } }; -} +} // end anonymous namespace static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory; @@ -75,34 +92,72 @@ const std::error_category &llvm::instrprof_category() { namespace llvm { +void SoftInstrProfErrors::addError(instrprof_error IE) { + if (IE == instrprof_error::success) + return; + + if (FirstError == instrprof_error::success) + FirstError = IE; + + switch (IE) { + case instrprof_error::hash_mismatch: + ++NumHashMismatches; + break; + case instrprof_error::count_mismatch: + ++NumCountMismatches; + break; + case instrprof_error::counter_overflow: + ++NumCounterOverflows; + break; + case instrprof_error::value_site_count_mismatch: + ++NumValueSiteCountMismatches; + break; + default: + llvm_unreachable("Not a soft error"); + } +} + +std::string InstrProfError::message() const { + return getInstrProfErrString(Err); +} + +char InstrProfError::ID = 0; + std::string getPGOFuncName(StringRef RawFuncName, GlobalValue::LinkageTypes Linkage, StringRef FileName, uint64_t Version LLVM_ATTRIBUTE_UNUSED) { + return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); +} + +// Return the PGOFuncName. This function has some special handling when called +// in LTO optimization. The following only applies when calling in LTO passes +// (when \c InLTO is true): LTO's internalization privatizes many global linkage +// symbols. This happens after value profile annotation, but those internal +// linkage functions should not have a source prefix. +// To differentiate compiler generated internal symbols from original ones, +// PGOFuncName meta data are created and attached to the original internal +// symbols in the value profile annotation step +// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta +// data, its original linkage must be non-internal. +std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { + if (!InLTO) { + StringRef FileName = (StaticFuncFullModulePrefix + ? F.getParent()->getName() + : sys::path::filename(F.getParent()->getName())); + return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); + } - // Function names may be prefixed with a binary '1' to indicate - // that the backend should not modify the symbols due to any platform - // naming convention. Do not include that '1' in the PGO profile name. - if (RawFuncName[0] == '\1') - RawFuncName = RawFuncName.substr(1); - - std::string FuncName = RawFuncName; - if (llvm::GlobalValue::isLocalLinkage(Linkage)) { - // For local symbols, prepend the main file name to distinguish them. - // Do not include the full path in the file name since there's no guarantee - // that it will stay the same, e.g., if the files are checked out from - // version control in different locations. - if (FileName.empty()) - FuncName = FuncName.insert(0, "<unknown>:"); - else - FuncName = FuncName.insert(0, FileName.str() + ":"); + // In LTO mode (when InLTO is true), first check if there is a meta data. + if (MDNode *MD = getPGOFuncNameMetadata(F)) { + StringRef S = cast<MDString>(MD->getOperand(0))->getString(); + return S.str(); } - return FuncName; -} -std::string getPGOFuncName(const Function &F, uint64_t Version) { - return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(), - Version); + // If there is no meta data, the function must be a global before the value + // profile annotation pass. Its current linkage may be internal if it is + // internalized in LTO mode. + return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { @@ -116,8 +171,8 @@ StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { // \p FuncName is the string used as profile lookup key for the function. A // symbol is created to hold the name. Return the legalized symbol name. -static std::string getPGOFuncNameVarName(StringRef FuncName, - GlobalValue::LinkageTypes Linkage) { +std::string getPGOFuncNameVarName(StringRef FuncName, + GlobalValue::LinkageTypes Linkage) { std::string VarName = getInstrProfNameVarPrefix(); VarName += FuncName; @@ -125,7 +180,7 @@ static std::string getPGOFuncNameVarName(StringRef FuncName, return VarName; // Now fix up illegal chars in local VarName that may upset the assembler. - const char *InvalidChars = "-:<>\"'"; + const char *InvalidChars = "-:<>/\"'"; size_t found = VarName.find_first_of(InvalidChars); while (found != std::string::npos) { VarName[found] = '_'; @@ -136,7 +191,7 @@ static std::string getPGOFuncNameVarName(StringRef FuncName, GlobalVariable *createPGOFuncNameVar(Module &M, GlobalValue::LinkageTypes Linkage, - StringRef FuncName) { + StringRef PGOFuncName) { // We generally want to match the function's linkage, but available_externally // and extern_weak both have the wrong semantics, and anything that doesn't @@ -149,10 +204,11 @@ GlobalVariable *createPGOFuncNameVar(Module &M, Linkage == GlobalValue::ExternalLinkage) Linkage = GlobalValue::PrivateLinkage; - auto *Value = ConstantDataArray::getString(M.getContext(), FuncName, false); + auto *Value = + ConstantDataArray::getString(M.getContext(), PGOFuncName, false); auto FuncNameVar = new GlobalVariable(M, Value->getType(), true, Linkage, Value, - getPGOFuncNameVarName(FuncName, Linkage)); + getPGOFuncNameVarName(PGOFuncName, Linkage)); // Hide the symbol so that we correctly get a copy for each executable. if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) @@ -161,63 +217,83 @@ GlobalVariable *createPGOFuncNameVar(Module &M, return FuncNameVar; } -GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) { - return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName); +GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { + return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); +} + +void InstrProfSymtab::create(Module &M, bool InLTO) { + for (Function &F : M) { + // Function may not have a name: like using asm("") to overwrite the name. + // Ignore in this case. + if (!F.hasName()) + continue; + const std::string &PGOFuncName = getPGOFuncName(F, InLTO); + addFuncName(PGOFuncName); + MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); + } + + finalizeSymtab(); } -int collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs, - bool doCompression, std::string &Result) { +Error collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs, + bool doCompression, std::string &Result) { + assert(NameStrs.size() && "No name data to emit"); + uint8_t Header[16], *P = Header; std::string UncompressedNameStrings = - join(NameStrs.begin(), NameStrs.end(), StringRef(" ")); + join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); + + assert(StringRef(UncompressedNameStrings) + .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && + "PGO name is invalid (contains separator token)"); unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); P += EncLen; - auto WriteStringToResult = [&](size_t CompressedLen, - const std::string &InputStr) { + auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { EncLen = encodeULEB128(CompressedLen, P); P += EncLen; char *HeaderStr = reinterpret_cast<char *>(&Header[0]); unsigned HeaderLen = P - &Header[0]; Result.append(HeaderStr, HeaderLen); Result += InputStr; - return 0; + return Error::success(); }; - if (!doCompression) + if (!doCompression) { return WriteStringToResult(0, UncompressedNameStrings); + } - SmallVector<char, 128> CompressedNameStrings; + SmallString<128> CompressedNameStrings; zlib::Status Success = zlib::compress(StringRef(UncompressedNameStrings), CompressedNameStrings, zlib::BestSizeCompression); if (Success != zlib::StatusOK) - return 1; + return make_error<InstrProfError>(instrprof_error::compress_failed); - return WriteStringToResult( - CompressedNameStrings.size(), - std::string(CompressedNameStrings.data(), CompressedNameStrings.size())); + return WriteStringToResult(CompressedNameStrings.size(), + CompressedNameStrings); } -StringRef getPGOFuncNameInitializer(GlobalVariable *NameVar) { +StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer()); StringRef NameStr = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); return NameStr; } -int collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars, - std::string &Result) { +Error collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars, + std::string &Result, bool doCompression) { std::vector<std::string> NameStrs; for (auto *NameVar : NameVars) { - NameStrs.push_back(getPGOFuncNameInitializer(NameVar)); + NameStrs.push_back(getPGOFuncNameVarInitializer(NameVar)); } - return collectPGOFuncNameStrings(NameStrs, zlib::isAvailable(), Result); + return collectPGOFuncNameStrings( + NameStrs, zlib::isAvailable() && doCompression, Result); } -int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { +Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { const uint8_t *P = reinterpret_cast<const uint8_t *>(NameStrings.data()); const uint8_t *EndP = reinterpret_cast<const uint8_t *>(NameStrings.data() + NameStrings.size()); @@ -235,7 +311,7 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { CompressedSize); if (zlib::uncompress(CompressedNameStrings, UncompressedNameStrings, UncompressedSize) != zlib::StatusOK) - return 1; + return make_error<InstrProfError>(instrprof_error::uncompress_failed); P += CompressedSize; NameStrings = StringRef(UncompressedNameStrings.data(), UncompressedNameStrings.size()); @@ -246,7 +322,7 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { } // Now parse the name strings. SmallVector<StringRef, 0> Names; - NameStrings.split(Names, ' '); + NameStrings.split(Names, getInstrProfNameSeparator()); for (StringRef &Name : Names) Symtab.addFuncName(Name); @@ -254,16 +330,16 @@ int readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { P++; } Symtab.finalizeSymtab(); - return 0; + return Error::success(); } -instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, - uint64_t Weight) { +void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE, + InstrProfValueSiteRecord &Input, + uint64_t Weight) { this->sortByTargetValues(); Input.sortByTargetValues(); auto I = ValueData.begin(); auto IE = ValueData.end(); - instrprof_error Result = instrprof_error::success; for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE; ++J) { while (I != IE && I->Value < J->Value) @@ -272,92 +348,80 @@ instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, bool Overflowed; I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed); if (Overflowed) - Result = instrprof_error::counter_overflow; + SIPE.addError(instrprof_error::counter_overflow); ++I; continue; } ValueData.insert(I, *J); } - return Result; } -instrprof_error InstrProfValueSiteRecord::scale(uint64_t Weight) { - instrprof_error Result = instrprof_error::success; +void InstrProfValueSiteRecord::scale(SoftInstrProfErrors &SIPE, + uint64_t Weight) { for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) { bool Overflowed; I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed); if (Overflowed) - Result = instrprof_error::counter_overflow; + SIPE.addError(instrprof_error::counter_overflow); } - return Result; } // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. -instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind, - InstrProfRecord &Src, - uint64_t Weight) { +void InstrProfRecord::mergeValueProfData(uint32_t ValueKind, + InstrProfRecord &Src, + uint64_t Weight) { uint32_t ThisNumValueSites = getNumValueSites(ValueKind); uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); - if (ThisNumValueSites != OtherNumValueSites) - return instrprof_error::value_site_count_mismatch; + if (ThisNumValueSites != OtherNumValueSites) { + SIPE.addError(instrprof_error::value_site_count_mismatch); + return; + } std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = getValueSitesForKind(ValueKind); std::vector<InstrProfValueSiteRecord> &OtherSiteRecords = Src.getValueSitesForKind(ValueKind); - instrprof_error Result = instrprof_error::success; for (uint32_t I = 0; I < ThisNumValueSites; I++) - MergeResult(Result, ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight)); - return Result; + ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight); } -instrprof_error InstrProfRecord::merge(InstrProfRecord &Other, - uint64_t Weight) { +void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) { // If the number of counters doesn't match we either have bad data // or a hash collision. - if (Counts.size() != Other.Counts.size()) - return instrprof_error::count_mismatch; - - instrprof_error Result = instrprof_error::success; + if (Counts.size() != Other.Counts.size()) { + SIPE.addError(instrprof_error::count_mismatch); + return; + } for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { bool Overflowed; Counts[I] = SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); if (Overflowed) - Result = instrprof_error::counter_overflow; + SIPE.addError(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - MergeResult(Result, mergeValueProfData(Kind, Other, Weight)); - - return Result; + mergeValueProfData(Kind, Other, Weight); } -instrprof_error InstrProfRecord::scaleValueProfData(uint32_t ValueKind, - uint64_t Weight) { +void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) { uint32_t ThisNumValueSites = getNumValueSites(ValueKind); std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = getValueSitesForKind(ValueKind); - instrprof_error Result = instrprof_error::success; for (uint32_t I = 0; I < ThisNumValueSites; I++) - MergeResult(Result, ThisSiteRecords[I].scale(Weight)); - return Result; + ThisSiteRecords[I].scale(SIPE, Weight); } -instrprof_error InstrProfRecord::scale(uint64_t Weight) { - instrprof_error Result = instrprof_error::success; +void InstrProfRecord::scale(uint64_t Weight) { for (auto &Count : this->Counts) { bool Overflowed; Count = SaturatingMultiply(Count, Weight, &Overflowed); - if (Overflowed && Result == instrprof_error::success) { - Result = instrprof_error::counter_overflow; - } + if (Overflowed) + SIPE.addError(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - MergeResult(Result, scaleValueProfData(Kind, Weight)); - - return Result; + scaleValueProfData(Kind, Weight); } // Map indirect call target name hash to name string. @@ -371,8 +435,14 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, std::lower_bound(ValueMap->begin(), ValueMap->end(), Value, [](const std::pair<uint64_t, uint64_t> &LHS, uint64_t RHS) { return LHS.first < RHS; }); - if (Result != ValueMap->end()) + // Raw function pointer collected by value profiler may be from + // external functions that are not instrumented. They won't have + // mapping data to be used by the deserializer. Force the value to + // be 0 in this case. + if (Result != ValueMap->end() && Result->first == Value) Value = (uint64_t)Result->second; + else + Value = 0; break; } } @@ -388,7 +458,7 @@ void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, std::vector<InstrProfValueSiteRecord> &ValueSites = getValueSitesForKind(ValueKind); if (N == 0) - ValueSites.push_back(InstrProfValueSiteRecord()); + ValueSites.emplace_back(); else ValueSites.emplace_back(VData, VData + N); } @@ -422,10 +492,8 @@ uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, } void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, - uint32_t K, uint32_t S, - uint64_t (*Mapper)(uint32_t, uint64_t)) { - return reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite( - Dst, K, S, Mapper); + uint32_t K, uint32_t S) { + reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S); } ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { @@ -436,12 +504,12 @@ ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { } static ValueProfRecordClosure InstrProfRecordClosure = { - 0, + nullptr, getNumValueKindsInstrProf, getNumValueSitesInstrProf, getNumValueDataInstrProf, getNumValueDataForSiteInstrProf, - 0, + nullptr, getValueForSiteInstrProf, allocValueProfDataInstrProf}; @@ -526,45 +594,45 @@ static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { ValueProfData()); } -instrprof_error ValueProfData::checkIntegrity() { +Error ValueProfData::checkIntegrity() { if (NumValueKinds > IPVK_Last + 1) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); // Total size needs to be mulltiple of quadword size. if (TotalSize % sizeof(uint64_t)) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); ValueProfRecord *VR = getFirstValueProfRecord(this); for (uint32_t K = 0; K < this->NumValueKinds; K++) { if (VR->Kind > IPVK_Last) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); VR = getValueProfRecordNext(VR); if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); } - return instrprof_error::success; + return Error::success(); } -ErrorOr<std::unique_ptr<ValueProfData>> +Expected<std::unique_ptr<ValueProfData>> ValueProfData::getValueProfData(const unsigned char *D, const unsigned char *const BufferEnd, support::endianness Endianness) { using namespace support; if (D + sizeof(ValueProfData) > BufferEnd) - return instrprof_error::truncated; + return make_error<InstrProfError>(instrprof_error::truncated); const unsigned char *Header = D; uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness); if (D + TotalSize > BufferEnd) - return instrprof_error::too_large; + return make_error<InstrProfError>(instrprof_error::too_large); std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize); memcpy(VPD.get(), D, TotalSize); // Byte swap. VPD->swapBytesToHost(Endianness); - instrprof_error EC = VPD->checkIntegrity(); - if (EC != instrprof_error::success) - return EC; + Error E = VPD->checkIntegrity(); + if (E) + return std::move(E); return std::move(VPD); } @@ -599,4 +667,117 @@ void ValueProfData::swapBytesFromHost(support::endianness Endianness) { sys::swapByteOrder<uint32_t>(NumValueKinds); } +void annotateValueSite(Module &M, Instruction &Inst, + const InstrProfRecord &InstrProfR, + InstrProfValueKind ValueKind, uint32_t SiteIdx, + uint32_t MaxMDCount) { + uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); + if (!NV) + return; + + uint64_t Sum = 0; + std::unique_ptr<InstrProfValueData[]> VD = + InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); + + ArrayRef<InstrProfValueData> VDs(VD.get(), NV); + annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); +} + +void annotateValueSite(Module &M, Instruction &Inst, + ArrayRef<InstrProfValueData> VDs, + uint64_t Sum, InstrProfValueKind ValueKind, + uint32_t MaxMDCount) { + LLVMContext &Ctx = M.getContext(); + MDBuilder MDHelper(Ctx); + SmallVector<Metadata *, 3> Vals; + // Tag + Vals.push_back(MDHelper.createString("VP")); + // Value Kind + Vals.push_back(MDHelper.createConstant( + ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind))); + // Total Count + Vals.push_back( + MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum))); + + // Value Profile Data + uint32_t MDCount = MaxMDCount; + for (auto &VD : VDs) { + Vals.push_back(MDHelper.createConstant( + ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); + Vals.push_back(MDHelper.createConstant( + ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); + if (--MDCount == 0) + break; + } + Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); +} + +bool getValueProfDataFromInst(const Instruction &Inst, + InstrProfValueKind ValueKind, + uint32_t MaxNumValueData, + InstrProfValueData ValueData[], + uint32_t &ActualNumValueData, uint64_t &TotalC) { + MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); + if (!MD) + return false; + + unsigned NOps = MD->getNumOperands(); + + if (NOps < 5) + return false; + + // Operand 0 is a string tag "VP": + MDString *Tag = cast<MDString>(MD->getOperand(0)); + if (!Tag) + return false; + + if (!Tag->getString().equals("VP")) + return false; + + // Now check kind: + ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); + if (!KindInt) + return false; + if (KindInt->getZExtValue() != ValueKind) + return false; + + // Get total count + ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); + if (!TotalCInt) + return false; + TotalC = TotalCInt->getZExtValue(); + + ActualNumValueData = 0; + + for (unsigned I = 3; I < NOps; I += 2) { + if (ActualNumValueData >= MaxNumValueData) + break; + ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); + ConstantInt *Count = + mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1)); + if (!Value || !Count) + return false; + ValueData[ActualNumValueData].Value = Value->getZExtValue(); + ValueData[ActualNumValueData].Count = Count->getZExtValue(); + ActualNumValueData++; + } + return true; +} + +MDNode *getPGOFuncNameMetadata(const Function &F) { + return F.getMetadata(getPGOFuncNameMetadataName()); +} + +void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { + // Only for internal linkage functions. + if (PGOFuncName == F.getName()) + return; + // Don't create duplicated meta-data. + if (getPGOFuncNameMetadata(F)) + return; + LLVMContext &C = F.getContext(); + MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); + F.setMetadata(getPGOFuncNameMetadataName(), N); } + +} // end namespace llvm diff --git a/gnu/llvm/lib/ProfileData/InstrProfReader.cpp b/gnu/llvm/lib/ProfileData/InstrProfReader.cpp index 5e83456822f..81c13b35ce3 100644 --- a/gnu/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/gnu/llvm/lib/ProfileData/InstrProfReader.cpp @@ -18,33 +18,33 @@ using namespace llvm; -static ErrorOr<std::unique_ptr<MemoryBuffer>> -setupMemoryBuffer(std::string Path) { +static Expected<std::unique_ptr<MemoryBuffer>> +setupMemoryBuffer(const Twine &Path) { ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = MemoryBuffer::getFileOrSTDIN(Path); if (std::error_code EC = BufferOrErr.getError()) - return EC; + return errorCodeToError(EC); return std::move(BufferOrErr.get()); } -static std::error_code initializeReader(InstrProfReader &Reader) { +static Error initializeReader(InstrProfReader &Reader) { return Reader.readHeader(); } -ErrorOr<std::unique_ptr<InstrProfReader>> -InstrProfReader::create(std::string Path) { +Expected<std::unique_ptr<InstrProfReader>> +InstrProfReader::create(const Twine &Path) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path); - if (std::error_code EC = BufferOrError.getError()) - return EC; + if (Error E = BufferOrError.takeError()) + return std::move(E); return InstrProfReader::create(std::move(BufferOrError.get())); } -ErrorOr<std::unique_ptr<InstrProfReader>> +Expected<std::unique_ptr<InstrProfReader>> InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { // Sanity check the buffer. if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) - return instrprof_error::too_large; + return make_error<InstrProfError>(instrprof_error::too_large); std::unique_ptr<InstrProfReader> Result; // Create the reader. @@ -57,46 +57,49 @@ InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { else if (TextInstrProfReader::hasFormat(*Buffer)) Result.reset(new TextInstrProfReader(std::move(Buffer))); else - return instrprof_error::unrecognized_format; + return make_error<InstrProfError>(instrprof_error::unrecognized_format); // Initialize the reader and return the result. - if (std::error_code EC = initializeReader(*Result)) - return EC; + if (Error E = initializeReader(*Result)) + return std::move(E); return std::move(Result); } -ErrorOr<std::unique_ptr<IndexedInstrProfReader>> -IndexedInstrProfReader::create(std::string Path) { +Expected<std::unique_ptr<IndexedInstrProfReader>> +IndexedInstrProfReader::create(const Twine &Path) { // Set up the buffer to read. auto BufferOrError = setupMemoryBuffer(Path); - if (std::error_code EC = BufferOrError.getError()) - return EC; + if (Error E = BufferOrError.takeError()) + return std::move(E); return IndexedInstrProfReader::create(std::move(BufferOrError.get())); } -ErrorOr<std::unique_ptr<IndexedInstrProfReader>> +Expected<std::unique_ptr<IndexedInstrProfReader>> IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { // Sanity check the buffer. if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) - return instrprof_error::too_large; + return make_error<InstrProfError>(instrprof_error::too_large); // Create the reader. if (!IndexedInstrProfReader::hasFormat(*Buffer)) - return instrprof_error::bad_magic; + return make_error<InstrProfError>(instrprof_error::bad_magic); auto Result = llvm::make_unique<IndexedInstrProfReader>(std::move(Buffer)); // Initialize the reader and return the result. - if (std::error_code EC = initializeReader(*Result)) - return EC; + if (Error E = initializeReader(*Result)) + return std::move(E); return std::move(Result); } void InstrProfIterator::Increment() { - if (Reader->readNextRecord(Record)) + if (auto E = Reader->readNextRecord(Record)) { + // Handle errors in the reader. + InstrProfError::take(std::move(E)); *this = InstrProfIterator(); + } } bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) { @@ -109,12 +112,30 @@ bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) { [](char c) { return ::isprint(c) || ::isspace(c); }); } -std::error_code TextInstrProfReader::readHeader() { +// Read the profile variant flag from the header: ":FE" means this is a FE +// generated profile. ":IR" means this is an IR level profile. Other strings +// with a leading ':' will be reported an error format. +Error TextInstrProfReader::readHeader() { Symtab.reset(new InstrProfSymtab()); + bool IsIRInstr = false; + if (!Line->startswith(":")) { + IsIRLevelProfile = false; + return success(); + } + StringRef Str = (Line)->substr(1); + if (Str.equals_lower("ir")) + IsIRInstr = true; + else if (Str.equals_lower("fe")) + IsIRInstr = false; + else + return error(instrprof_error::bad_header); + + ++Line; + IsIRLevelProfile = IsIRInstr; return success(); } -std::error_code +Error TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { #define CHECK_LINE_END(Line) \ @@ -156,7 +177,7 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { std::vector<InstrProfValueData> CurrentValues; for (uint32_t V = 0; V < NumValueData; V++) { CHECK_LINE_END(Line); - std::pair<StringRef, StringRef> VD = Line->split(':'); + std::pair<StringRef, StringRef> VD = Line->rsplit(':'); uint64_t TakenCount, Value; if (VK == IPVK_IndirectCallTarget) { Symtab->addFuncName(VD.first); @@ -178,7 +199,7 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { #undef VP_READ_ADVANCE } -std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { +Error TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { // Skip empty lines and comments. while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) ++Line; @@ -220,8 +241,8 @@ std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { } // Check if value profile data exists and read it if so. - if (std::error_code EC = readValueProfileData(Record)) - return EC; + if (Error E = readValueProfileData(Record)) + return E; // This is needed to avoid two pass parsing because llvm-profdata // does dumping while reading. @@ -240,7 +261,7 @@ bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) { } template <class IntPtrT> -std::error_code RawInstrProfReader<IntPtrT>::readHeader() { +Error RawInstrProfReader<IntPtrT>::readHeader() { if (!hasFormat(*DataBuffer)) return error(instrprof_error::bad_magic); if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header)) @@ -252,26 +273,25 @@ std::error_code RawInstrProfReader<IntPtrT>::readHeader() { } template <class IntPtrT> -std::error_code -RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) { +Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) { const char *End = DataBuffer->getBufferEnd(); // Skip zero padding between profiles. while (CurrentPos != End && *CurrentPos == 0) ++CurrentPos; // If there's nothing left, we're done. if (CurrentPos == End) - return instrprof_error::eof; + return make_error<InstrProfError>(instrprof_error::eof); // If there isn't enough space for another header, this is probably just // garbage at the end of the file. if (CurrentPos + sizeof(RawInstrProf::Header) > End) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); // The writer ensures each profile is padded to start at an aligned address. if (reinterpret_cast<size_t>(CurrentPos) % alignOf<uint64_t>()) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); // The magic should have the same byte order as in the previous header. uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos); if (Magic != swap(RawInstrProf::getMagic<IntPtrT>())) - return instrprof_error::bad_magic; + return make_error<InstrProfError>(instrprof_error::bad_magic); // There's another profile to read, so we need to process the header. auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos); @@ -279,30 +299,31 @@ RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) { } template <class IntPtrT> -void RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) { +Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) { + if (Error E = Symtab.create(StringRef(NamesStart, NamesSize))) + return error(std::move(E)); for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) { - StringRef FunctionName(getName(I->NamePtr), swap(I->NameSize)); - Symtab.addFuncName(FunctionName); const IntPtrT FPtr = swap(I->FunctionPointer); if (!FPtr) continue; - Symtab.mapAddress(FPtr, IndexedInstrProf::ComputeHash(FunctionName)); + Symtab.mapAddress(FPtr, I->NameRef); } Symtab.finalizeSymtab(); + return success(); } template <class IntPtrT> -std::error_code -RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) { - if (swap(Header.Version) != RawInstrProf::Version) +Error RawInstrProfReader<IntPtrT>::readHeader( + const RawInstrProf::Header &Header) { + Version = swap(Header.Version); + if (GET_VERSION(Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); NamesDelta = swap(Header.NamesDelta); auto DataSize = swap(Header.DataSize); auto CountersSize = swap(Header.CountersSize); - auto NamesSize = swap(Header.NamesSize); - auto ValueDataSize = swap(Header.ValueDataSize); + NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>); @@ -312,10 +333,9 @@ RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) { ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes; ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; - size_t ProfileSize = ValueDataOffset + ValueDataSize; auto *Start = reinterpret_cast<const char *>(&Header); - if (Start + ProfileSize > DataBuffer->getBufferEnd()) + if (Start + ValueDataOffset > DataBuffer->getBufferEnd()) return error(instrprof_error::bad_header); Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>( @@ -324,33 +344,29 @@ RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) { CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset); NamesStart = Start + NamesOffset; ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset); - ProfileEnd = Start + ProfileSize; std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>(); - createSymtab(*NewSymtab.get()); + if (Error E = createSymtab(*NewSymtab.get())) + return E; + Symtab = std::move(NewSymtab); return success(); } template <class IntPtrT> -std::error_code RawInstrProfReader<IntPtrT>::readName(InstrProfRecord &Record) { - Record.Name = StringRef(getName(Data->NamePtr), swap(Data->NameSize)); - if (Record.Name.data() < NamesStart || - Record.Name.data() + Record.Name.size() > - reinterpret_cast<const char *>(ValueDataStart)) - return error(instrprof_error::malformed); +Error RawInstrProfReader<IntPtrT>::readName(InstrProfRecord &Record) { + Record.Name = getName(Data->NameRef); return success(); } template <class IntPtrT> -std::error_code RawInstrProfReader<IntPtrT>::readFuncHash( - InstrProfRecord &Record) { +Error RawInstrProfReader<IntPtrT>::readFuncHash(InstrProfRecord &Record) { Record.Hash = swap(Data->FuncHash); return success(); } template <class IntPtrT> -std::error_code RawInstrProfReader<IntPtrT>::readRawCounts( +Error RawInstrProfReader<IntPtrT>::readRawCounts( InstrProfRecord &Record) { uint32_t NumCounters = swap(Data->NumCounters); IntPtrT CounterPtr = Data->CounterPtr; @@ -377,8 +393,8 @@ std::error_code RawInstrProfReader<IntPtrT>::readRawCounts( } template <class IntPtrT> -std::error_code -RawInstrProfReader<IntPtrT>::readValueProfilingData(InstrProfRecord &Record) { +Error RawInstrProfReader<IntPtrT>::readValueProfilingData( + InstrProfRecord &Record) { Record.clearValueData(); CurValueDataSize = 0; @@ -390,41 +406,44 @@ RawInstrProfReader<IntPtrT>::readValueProfilingData(InstrProfRecord &Record) { if (!NumValueKinds) return success(); - ErrorOr<std::unique_ptr<ValueProfData>> VDataPtrOrErr = - ValueProfData::getValueProfData(ValueDataStart, - (const unsigned char *)ProfileEnd, - getDataEndianness()); + Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr = + ValueProfData::getValueProfData( + ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(), + getDataEndianness()); - if (VDataPtrOrErr.getError()) - return VDataPtrOrErr.getError(); + if (Error E = VDataPtrOrErr.takeError()) + return E; + // Note that besides deserialization, this also performs the conversion for + // indirect call targets. The function pointers from the raw profile are + // remapped into function name hashes. VDataPtrOrErr.get()->deserializeTo(Record, &Symtab->getAddrHashMap()); CurValueDataSize = VDataPtrOrErr.get()->getSize(); return success(); } template <class IntPtrT> -std::error_code -RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) { +Error RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) { if (atEnd()) - if (std::error_code EC = readNextHeader(ProfileEnd)) - return EC; + // At this point, ValueDataStart field points to the next header. + if (Error E = readNextHeader(getNextHeaderPos())) + return E; // Read name ad set it in Record. - if (std::error_code EC = readName(Record)) - return EC; + if (Error E = readName(Record)) + return E; // Read FuncHash and set it in Record. - if (std::error_code EC = readFuncHash(Record)) - return EC; + if (Error E = readFuncHash(Record)) + return E; // Read raw counts and set Record. - if (std::error_code EC = readRawCounts(Record)) - return EC; + if (Error E = readRawCounts(Record)) + return E; // Read value data and set Record. - if (std::error_code EC = readValueProfilingData(Record)) - return EC; + if (Error E = readValueProfilingData(Record)) + return E; // Iterate. advanceData(); @@ -446,10 +465,10 @@ typedef InstrProfLookupTrait::offset_type offset_type; bool InstrProfLookupTrait::readValueProfilingData( const unsigned char *&D, const unsigned char *const End) { - ErrorOr<std::unique_ptr<ValueProfData>> VDataPtrOrErr = + Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr = ValueProfData::getValueProfData(D, End, ValueProfDataEndianness); - if (VDataPtrOrErr.getError()) + if (VDataPtrOrErr.takeError()) return false; VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr); @@ -475,10 +494,10 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, return data_type(); uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D); - // Initialize number of counters for FormatVersion == 1. + // Initialize number of counters for GET_VERSION(FormatVersion) == 1. uint64_t CountsSize = N / sizeof(uint64_t) - 1; // If format version is different then read the number of counters. - if (FormatVersion != 1) { + if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) { if (D + sizeof(uint64_t) > End) return data_type(); CountsSize = endian::readNext<uint64_t, little, unaligned>(D); @@ -495,7 +514,8 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. - if (FormatVersion > 2 && !readValueProfilingData(D, End)) { + if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && + !readValueProfilingData(D, End)) { DataBuffer.clear(); return data_type(); } @@ -504,31 +524,31 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, } template <typename HashTableImpl> -std::error_code InstrProfReaderIndex<HashTableImpl>::getRecords( +Error InstrProfReaderIndex<HashTableImpl>::getRecords( StringRef FuncName, ArrayRef<InstrProfRecord> &Data) { auto Iter = HashTable->find(FuncName); if (Iter == HashTable->end()) - return instrprof_error::unknown_function; + return make_error<InstrProfError>(instrprof_error::unknown_function); Data = (*Iter); if (Data.empty()) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); - return instrprof_error::success; + return Error::success(); } template <typename HashTableImpl> -std::error_code InstrProfReaderIndex<HashTableImpl>::getRecords( +Error InstrProfReaderIndex<HashTableImpl>::getRecords( ArrayRef<InstrProfRecord> &Data) { if (atEnd()) - return instrprof_error::eof; + return make_error<InstrProfError>(instrprof_error::eof); Data = *RecordIterator; if (Data.empty()) - return instrprof_error::malformed; + return make_error<InstrProfError>(instrprof_error::malformed); - return instrprof_error::success; + return Error::success(); } template <typename HashTableImpl> @@ -553,7 +573,56 @@ bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { return Magic == IndexedInstrProf::Magic; } -std::error_code IndexedInstrProfReader::readHeader() { +const unsigned char * +IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version, + const unsigned char *Cur) { + using namespace IndexedInstrProf; + using namespace support; + if (Version >= IndexedInstrProf::Version4) { + const IndexedInstrProf::Summary *SummaryInLE = + reinterpret_cast<const IndexedInstrProf::Summary *>(Cur); + uint64_t NFields = + endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields); + uint64_t NEntries = + endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries); + uint32_t SummarySize = + IndexedInstrProf::Summary::getSize(NFields, NEntries); + std::unique_ptr<IndexedInstrProf::Summary> SummaryData = + IndexedInstrProf::allocSummary(SummarySize); + + const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE); + uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get()); + for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) + Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]); + + llvm::SummaryEntryVector DetailedSummary; + for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) { + const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I); + DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount, + Ent.NumBlocks); + } + // initialize InstrProfSummary using the SummaryData from disk. + this->Summary = llvm::make_unique<ProfileSummary>( + ProfileSummary::PSK_Instr, DetailedSummary, + SummaryData->get(Summary::TotalBlockCount), + SummaryData->get(Summary::MaxBlockCount), + SummaryData->get(Summary::MaxInternalBlockCount), + SummaryData->get(Summary::MaxFunctionCount), + SummaryData->get(Summary::TotalNumBlocks), + SummaryData->get(Summary::TotalNumFunctions)); + return Cur + SummarySize; + } else { + // For older version of profile data, we need to compute on the fly: + using namespace IndexedInstrProf; + InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); + // FIXME: This only computes an empty summary. Need to call addRecord for + // all InstrProfRecords to get the correct summary. + this->Summary = Builder.getSummary(); + return Cur; + } +} + +Error IndexedInstrProfReader::readHeader() { const unsigned char *Start = (const unsigned char *)DataBuffer->getBufferStart(); const unsigned char *Cur = Start; @@ -572,12 +641,11 @@ std::error_code IndexedInstrProfReader::readHeader() { // Read the version. uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version); - if (FormatVersion > IndexedInstrProf::Version) + if (GET_VERSION(FormatVersion) > + IndexedInstrProf::ProfVersion::CurrentVersion) return error(instrprof_error::unsupported_version); - // Read the maximal function count. - MaxFunctionCount = - endian::byte_swap<uint64_t, little>(Header->MaxFunctionCount); + Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur); // Read the hash type and start offset. IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>( @@ -606,13 +674,13 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() { return *Symtab.get(); } -ErrorOr<InstrProfRecord> +Expected<InstrProfRecord> IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName, uint64_t FuncHash) { ArrayRef<InstrProfRecord> Data; - std::error_code EC = Index->getRecords(FuncName, Data); - if (EC != instrprof_error::success) - return EC; + Error Err = Index->getRecords(FuncName, Data); + if (Err) + return std::move(Err); // Found it. Look for counters with the right hash. for (unsigned I = 0, E = Data.size(); I < E; ++I) { // Check for a match and fill the vector if there is one. @@ -623,26 +691,25 @@ IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName, return error(instrprof_error::hash_mismatch); } -std::error_code -IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, uint64_t FuncHash, - std::vector<uint64_t> &Counts) { - ErrorOr<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash); - if (std::error_code EC = Record.getError()) - return EC; +Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, + uint64_t FuncHash, + std::vector<uint64_t> &Counts) { + Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash); + if (Error E = Record.takeError()) + return error(std::move(E)); Counts = Record.get().Counts; return success(); } -std::error_code IndexedInstrProfReader::readNextRecord( - InstrProfRecord &Record) { +Error IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) { static unsigned RecordIndex = 0; ArrayRef<InstrProfRecord> Data; - std::error_code EC = Index->getRecords(Data); - if (EC != instrprof_error::success) - return error(EC); + Error E = Index->getRecords(Data); + if (E) + return error(std::move(E)); Record = Data[RecordIndex++]; if (RecordIndex >= Data.size()) { diff --git a/gnu/llvm/lib/ProfileData/InstrProfWriter.cpp b/gnu/llvm/lib/ProfileData/InstrProfWriter.cpp index f5227248af2..e25299ef670 100644 --- a/gnu/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/gnu/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -20,10 +20,59 @@ using namespace llvm; -namespace { -static support::endianness ValueProfDataEndianness = support::little; +// A struct to define how the data stream should be patched. For Indexed +// profiling, only uint64_t data type is needed. +struct PatchItem { + uint64_t Pos; // Where to patch. + uint64_t *D; // Pointer to an array of source data. + int N; // Number of elements in \c D array. +}; + +namespace llvm { +// A wrapper class to abstract writer stream with support of bytes +// back patching. +class ProfOStream { + +public: + ProfOStream(llvm::raw_fd_ostream &FD) : IsFDOStream(true), OS(FD), LE(FD) {} + ProfOStream(llvm::raw_string_ostream &STR) + : IsFDOStream(false), OS(STR), LE(STR) {} + + uint64_t tell() { return OS.tell(); } + void write(uint64_t V) { LE.write<uint64_t>(V); } + // \c patch can only be called when all data is written and flushed. + // For raw_string_ostream, the patch is done on the target string + // directly and it won't be reflected in the stream's internal buffer. + void patch(PatchItem *P, int NItems) { + using namespace support; + if (IsFDOStream) { + llvm::raw_fd_ostream &FDOStream = static_cast<llvm::raw_fd_ostream &>(OS); + for (int K = 0; K < NItems; K++) { + FDOStream.seek(P[K].Pos); + for (int I = 0; I < P[K].N; I++) + write(P[K].D[I]); + } + } else { + llvm::raw_string_ostream &SOStream = + static_cast<llvm::raw_string_ostream &>(OS); + std::string &Data = SOStream.str(); // with flush + for (int K = 0; K < NItems; K++) { + for (int I = 0; I < P[K].N; I++) { + uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]); + Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t), + (const char *)&Bytes, sizeof(uint64_t)); + } + } + } + } + // If \c OS is an instance of \c raw_fd_ostream, this field will be + // true. Otherwise, \c OS will be an raw_string_ostream. + bool IsFDOStream; + raw_ostream &OS; + support::endian::Writer<support::little> LE; +}; -class InstrProfRecordTrait { +class InstrProfRecordWriterTrait { public: typedef StringRef key_type; typedef StringRef key_type_ref; @@ -34,6 +83,10 @@ public: typedef uint64_t hash_value_type; typedef uint64_t offset_type; + support::endianness ValueProfDataEndianness; + InstrProfSummaryBuilder *SummaryBuilder; + + InstrProfRecordWriterTrait() : ValueProfDataEndianness(support::little) {} static hash_value_type ComputeHash(key_type_ref K) { return IndexedInstrProf::ComputeHash(K); } @@ -61,16 +114,16 @@ public: return std::make_pair(N, M); } - static void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N){ + void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) { Out.write(K.data(), N); } - static void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, - offset_type) { + void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) { using namespace llvm::support; endian::Writer<little> LE(Out); for (const auto &ProfileData : *V) { const InstrProfRecord &ProfRecord = ProfileData.second; + SummaryBuilder->addRecord(ProfRecord); LE.write<uint64_t>(ProfileData.first); // Function hash LE.write<uint64_t>(ProfRecord.Counts.size()); @@ -88,14 +141,22 @@ public: }; } +InstrProfWriter::InstrProfWriter(bool Sparse) + : Sparse(Sparse), FunctionData(), ProfileKind(PF_Unknown), + InfoObj(new InstrProfRecordWriterTrait()) {} + +InstrProfWriter::~InstrProfWriter() { delete InfoObj; } + // Internal interface for testing purpose only. void InstrProfWriter::setValueProfDataEndianness( support::endianness Endianness) { - ValueProfDataEndianness = Endianness; + InfoObj->ValueProfDataEndianness = Endianness; +} +void InstrProfWriter::setOutputSparse(bool Sparse) { + this->Sparse = Sparse; } -std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I, - uint64_t Weight) { +Error InstrProfWriter::addRecord(InstrProfRecord &&I, uint64_t Weight) { auto &ProfileDataMap = FunctionData[I.Name]; bool NewFunc; @@ -104,73 +165,128 @@ std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I, ProfileDataMap.insert(std::make_pair(I.Hash, InstrProfRecord())); InstrProfRecord &Dest = Where->second; - instrprof_error Result = instrprof_error::success; if (NewFunc) { // We've never seen a function with this name and hash, add it. Dest = std::move(I); // Fix up the name to avoid dangling reference. Dest.Name = FunctionData.find(Dest.Name)->getKey(); if (Weight > 1) - Result = Dest.scale(Weight); + Dest.scale(Weight); } else { // We're updating a function we've seen before. - Result = Dest.merge(I, Weight); + Dest.merge(I, Weight); } Dest.sortValueData(); - // We keep track of the max function count as we go for simplicity. - // Update this statistic no matter the result of the merge. - if (Dest.Counts[0] > MaxFunctionCount) - MaxFunctionCount = Dest.Counts[0]; + return Dest.takeError(); +} + +bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) { + if (!Sparse) + return true; + for (const auto &Func : PD) { + const InstrProfRecord &IPR = Func.second; + if (std::any_of(IPR.Counts.begin(), IPR.Counts.end(), + [](uint64_t Count) { return Count > 0; })) + return true; + } + return false; +} - return Result; +static void setSummary(IndexedInstrProf::Summary *TheSummary, + ProfileSummary &PS) { + using namespace IndexedInstrProf; + std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary(); + TheSummary->NumSummaryFields = Summary::NumKinds; + TheSummary->NumCutoffEntries = Res.size(); + TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount()); + TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount()); + TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount()); + TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount()); + TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts()); + TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions()); + for (unsigned I = 0; I < Res.size(); I++) + TheSummary->setEntry(I, Res[I]); } -std::pair<uint64_t, uint64_t> InstrProfWriter::writeImpl(raw_ostream &OS) { - OnDiskChainedHashTableGenerator<InstrProfRecordTrait> Generator; +void InstrProfWriter::writeImpl(ProfOStream &OS) { + OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator; + + using namespace IndexedInstrProf; + InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs); + InfoObj->SummaryBuilder = &ISB; // Populate the hash table generator. for (const auto &I : FunctionData) - Generator.insert(I.getKey(), &I.getValue()); - - using namespace llvm::support; - endian::Writer<little> LE(OS); - + if (shouldEncodeData(I.getValue())) + Generator.insert(I.getKey(), &I.getValue()); // Write the header. IndexedInstrProf::Header Header; Header.Magic = IndexedInstrProf::Magic; - Header.Version = IndexedInstrProf::Version; - Header.MaxFunctionCount = MaxFunctionCount; + Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion; + if (ProfileKind == PF_IRLevel) + Header.Version |= VARIANT_MASK_IR_PROF; + Header.Unused = 0; Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType); Header.HashOffset = 0; int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t); - // Only write out all the fields execpt 'HashOffset'. We need + // Only write out all the fields except 'HashOffset'. We need // to remember the offset of that field to allow back patching // later. for (int I = 0; I < N - 1; I++) - LE.write<uint64_t>(reinterpret_cast<uint64_t *>(&Header)[I]); + OS.write(reinterpret_cast<uint64_t *>(&Header)[I]); - // Save a space to write the hash table start location. - uint64_t HashTableStartLoc = OS.tell(); + // Save the location of Header.HashOffset field in \c OS. + uint64_t HashTableStartFieldOffset = OS.tell(); // Reserve the space for HashOffset field. - LE.write<uint64_t>(0); - // Write the hash table. - uint64_t HashTableStart = Generator.Emit(OS); + OS.write(0); + + // Reserve space to write profile summary data. + uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size(); + uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); + // Remember the summary offset. + uint64_t SummaryOffset = OS.tell(); + for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) + OS.write(0); - return std::make_pair(HashTableStartLoc, HashTableStart); + // Write the hash table. + uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj); + + // Allocate space for data to be serialized out. + std::unique_ptr<IndexedInstrProf::Summary> TheSummary = + IndexedInstrProf::allocSummary(SummarySize); + // Compute the Summary and copy the data to the data + // structure to be serialized out (to disk or buffer). + std::unique_ptr<ProfileSummary> PS = ISB.getSummary(); + setSummary(TheSummary.get(), *PS); + InfoObj->SummaryBuilder = 0; + + // Now do the final patch: + PatchItem PatchItems[] = { + // Patch the Header.HashOffset field. + {HashTableStartFieldOffset, &HashTableStart, 1}, + // Patch the summary data. + {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()), + (int)(SummarySize / sizeof(uint64_t))}}; + OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems)); } void InstrProfWriter::write(raw_fd_ostream &OS) { // Write the hash table. - auto TableStart = writeImpl(OS); + ProfOStream POS(OS); + writeImpl(POS); +} - // Go back and fill in the hash table start. - using namespace support; - OS.seek(TableStart.first); - // Now patch the HashOffset field previously reserved. - endian::Writer<little>(OS).write<uint64_t>(TableStart.second); +std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() { + std::string Data; + llvm::raw_string_ostream OS(Data); + ProfOStream POS(OS); + // Write the hash table. + writeImpl(POS); + // Return this in an aligned memory buffer. + return MemoryBuffer::getMemBufferCopy(Data); } static const char *ValueProfKindStr[] = { @@ -218,29 +334,16 @@ void InstrProfWriter::writeRecordInText(const InstrProfRecord &Func, } void InstrProfWriter::writeText(raw_fd_ostream &OS) { + if (ProfileKind == PF_IRLevel) + OS << "# IR level Instrumentation Flag\n:ir\n"; InstrProfSymtab Symtab; for (const auto &I : FunctionData) - Symtab.addFuncName(I.getKey()); + if (shouldEncodeData(I.getValue())) + Symtab.addFuncName(I.getKey()); Symtab.finalizeSymtab(); for (const auto &I : FunctionData) - for (const auto &Func : I.getValue()) - writeRecordInText(Func.second, Symtab, OS); -} - -std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() { - std::string Data; - llvm::raw_string_ostream OS(Data); - // Write the hash table. - auto TableStart = writeImpl(OS); - OS.flush(); - - // Go back and fill in the hash table start. - using namespace support; - uint64_t Bytes = endian::byte_swap<uint64_t, little>(TableStart.second); - Data.replace(TableStart.first, sizeof(uint64_t), (const char *)&Bytes, - sizeof(uint64_t)); - - // Return this in an aligned memory buffer. - return MemoryBuffer::getMemBufferCopy(Data); + if (shouldEncodeData(I.getValue())) + for (const auto &Func : I.getValue()) + writeRecordInText(Func.second, Symtab, OS); } diff --git a/gnu/llvm/lib/ProfileData/LLVMBuild.txt b/gnu/llvm/lib/ProfileData/LLVMBuild.txt index a7f471fc582..b3d749feda6 100644 --- a/gnu/llvm/lib/ProfileData/LLVMBuild.txt +++ b/gnu/llvm/lib/ProfileData/LLVMBuild.txt @@ -15,8 +15,11 @@ ; ;===------------------------------------------------------------------------===; +[common] +subdirectories = Coverage + [component_0] type = Library name = ProfileData parent = Libraries -required_libraries = Core Support Object +required_libraries = Core Support diff --git a/gnu/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/gnu/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp new file mode 100644 index 00000000000..f8c3717007b --- /dev/null +++ b/gnu/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -0,0 +1,116 @@ +//=-- ProfilesummaryBuilder.cpp - Profile summary computation ---------------=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for computing profile summary data. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/Attributes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Type.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/Casting.h" + +using namespace llvm; + +// A set of cutoff values. Each value, when divided by ProfileSummary::Scale +// (which is 1000000) is a desired percentile of total counts. +static const uint32_t DefaultCutoffsData[] = { + 10000, /* 1% */ + 100000, /* 10% */ + 200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000, + 800000, 900000, 950000, 990000, 999000, 999900, 999990, 999999}; +const ArrayRef<uint32_t> ProfileSummaryBuilder::DefaultCutoffs = + DefaultCutoffsData; + +void InstrProfSummaryBuilder::addRecord(const InstrProfRecord &R) { + // The first counter is not necessarily an entry count for IR + // instrumentation profiles. + // Eventually MaxFunctionCount will become obsolete and this can be + // removed. + addEntryCount(R.Counts[0]); + for (size_t I = 1, E = R.Counts.size(); I < E; ++I) + addInternalCount(R.Counts[I]); +} + +// To compute the detailed summary, we consider each line containing samples as +// equivalent to a block with a count in the instrumented profile. +void SampleProfileSummaryBuilder::addRecord( + const sampleprof::FunctionSamples &FS) { + NumFunctions++; + if (FS.getHeadSamples() > MaxFunctionCount) + MaxFunctionCount = FS.getHeadSamples(); + for (const auto &I : FS.getBodySamples()) + addCount(I.second.getSamples()); +} + +// The argument to this method is a vector of cutoff percentages and the return +// value is a vector of (Cutoff, MinCount, NumCounts) triplets. +void ProfileSummaryBuilder::computeDetailedSummary() { + if (DetailedSummaryCutoffs.empty()) + return; + auto Iter = CountFrequencies.begin(); + auto End = CountFrequencies.end(); + std::sort(DetailedSummaryCutoffs.begin(), DetailedSummaryCutoffs.end()); + + uint32_t CountsSeen = 0; + uint64_t CurrSum = 0, Count = 0; + + for (uint32_t Cutoff : DetailedSummaryCutoffs) { + assert(Cutoff <= 999999); + APInt Temp(128, TotalCount); + APInt N(128, Cutoff); + APInt D(128, ProfileSummary::Scale); + Temp *= N; + Temp = Temp.sdiv(D); + uint64_t DesiredCount = Temp.getZExtValue(); + assert(DesiredCount <= TotalCount); + while (CurrSum < DesiredCount && Iter != End) { + Count = Iter->first; + uint32_t Freq = Iter->second; + CurrSum += (Count * Freq); + CountsSeen += Freq; + Iter++; + } + assert(CurrSum >= DesiredCount); + ProfileSummaryEntry PSE = {Cutoff, Count, CountsSeen}; + DetailedSummary.push_back(PSE); + } +} + +std::unique_ptr<ProfileSummary> SampleProfileSummaryBuilder::getSummary() { + computeDetailedSummary(); + return llvm::make_unique<ProfileSummary>( + ProfileSummary::PSK_Sample, DetailedSummary, TotalCount, MaxCount, 0, + MaxFunctionCount, NumCounts, NumFunctions); +} + +std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() { + computeDetailedSummary(); + return llvm::make_unique<ProfileSummary>( + ProfileSummary::PSK_Instr, DetailedSummary, TotalCount, MaxCount, + MaxInternalBlockCount, MaxFunctionCount, NumCounts, NumFunctions); +} + +void InstrProfSummaryBuilder::addEntryCount(uint64_t Count) { + addCount(Count); + NumFunctions++; + if (Count > MaxFunctionCount) + MaxFunctionCount = Count; +} + +void InstrProfSummaryBuilder::addInternalCount(uint64_t Count) { + addCount(Count); + if (Count > MaxInternalBlockCount) + MaxInternalBlockCount = Count; +} diff --git a/gnu/llvm/lib/ProfileData/SampleProf.cpp b/gnu/llvm/lib/ProfileData/SampleProf.cpp index 9ded757f2b2..cb0246113d8 100644 --- a/gnu/llvm/lib/ProfileData/SampleProf.cpp +++ b/gnu/llvm/lib/ProfileData/SampleProf.cpp @@ -20,6 +20,9 @@ using namespace llvm::sampleprof; using namespace llvm; namespace { +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. class SampleProfErrorCategoryType : public std::error_category { const char *name() const LLVM_NOEXCEPT override { return "llvm.sampleprof"; } std::string message(int IE) const override { @@ -71,20 +74,7 @@ raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS, return OS; } -void LineLocation::dump() const { print(dbgs()); } - -void CallsiteLocation::print(raw_ostream &OS) const { - LineLocation::print(OS); - OS << ": inlined callee: " << CalleeName; -} - -void CallsiteLocation::dump() const { print(dbgs()); } - -inline raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS, - const CallsiteLocation &Loc) { - Loc.print(OS); - return OS; -} +LLVM_DUMP_METHOD void LineLocation::dump() const { print(dbgs()); } /// \brief Print the sample record to the stream \p OS indented by \p Indent. void SampleRecord::print(raw_ostream &OS, unsigned Indent) const { @@ -97,7 +87,7 @@ void SampleRecord::print(raw_ostream &OS, unsigned Indent) const { OS << "\n"; } -void SampleRecord::dump() const { print(dbgs(), 0); } +LLVM_DUMP_METHOD void SampleRecord::dump() const { print(dbgs(), 0); } raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS, const SampleRecord &Sample) { @@ -127,11 +117,11 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { OS.indent(Indent); if (CallsiteSamples.size() > 0) { OS << "Samples collected in inlined callsites {\n"; - SampleSorter<CallsiteLocation, FunctionSamples> SortedCallsiteSamples( + SampleSorter<LineLocation, FunctionSamples> SortedCallsiteSamples( CallsiteSamples); for (const auto &CS : SortedCallsiteSamples.get()) { OS.indent(Indent + 2); - OS << CS->first << ": "; + OS << CS->first << ": inlined callee: " << CS->second.getName() << ": "; CS->second.print(OS, Indent + 4); } OS << "}\n"; diff --git a/gnu/llvm/lib/ProfileData/SampleProfReader.cpp b/gnu/llvm/lib/ProfileData/SampleProfReader.cpp index 93cd87bb82f..af80b036a5b 100644 --- a/gnu/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/gnu/llvm/lib/ProfileData/SampleProfReader.cpp @@ -22,7 +22,7 @@ #include "llvm/ProfileData/SampleProfReader.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/LEB128.h" @@ -68,11 +68,8 @@ static bool ParseHead(const StringRef &Input, StringRef &FName, return true; } - /// \brief Returns true if line offset \p L is legal (only has 16 bits). -static bool isOffsetLegal(unsigned L) { - return (L & 0xffff) == L; -} +static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } /// \brief Parse \p Input as line sample. /// @@ -180,6 +177,7 @@ std::error_code SampleProfileReaderText::read() { } Profiles[FName] = FunctionSamples(); FunctionSamples &FProfile = Profiles[FName]; + FProfile.setName(FName); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); InlineStack.clear(); @@ -202,7 +200,8 @@ std::error_code SampleProfileReaderText::read() { InlineStack.pop_back(); } FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( - CallsiteLocation(LineOffset, Discriminator, FName)); + LineLocation(LineOffset, Discriminator)); + FSamples.setName(FName); MergeResult(Result, FSamples.addTotalSamples(NumSamples)); InlineStack.push_back(&FSamples); } else { @@ -220,6 +219,8 @@ std::error_code SampleProfileReaderText::read() { } } } + if (Result == sampleprof_error::success) + computeSummary(); return Result; } @@ -351,8 +352,9 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { if (std::error_code EC = FName.getError()) return EC; - FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( - CallsiteLocation(*LineOffset, *Discriminator, *FName)); + FunctionSamples &CalleeProfile = + FProfile.functionSamplesAt(LineLocation(*LineOffset, *Discriminator)); + CalleeProfile.setName(*FName); if (std::error_code EC = readProfile(CalleeProfile)) return EC; } @@ -372,6 +374,7 @@ std::error_code SampleProfileReaderBinary::read() { Profiles[*FName] = FunctionSamples(); FunctionSamples &FProfile = Profiles[*FName]; + FProfile.setName(*FName); FProfile.addHeadSamples(*NumHeadSamples); @@ -400,6 +403,9 @@ std::error_code SampleProfileReaderBinary::readHeader() { else if (*Version != SPVersion()) return sampleprof_error::unsupported_version; + if (std::error_code EC = readSummary()) + return EC; + // Read the name table. auto Size = readNumber<uint32_t>(); if (std::error_code EC = Size.getError()) @@ -415,6 +421,62 @@ std::error_code SampleProfileReaderBinary::readHeader() { return sampleprof_error::success; } +std::error_code SampleProfileReaderBinary::readSummaryEntry( + std::vector<ProfileSummaryEntry> &Entries) { + auto Cutoff = readNumber<uint64_t>(); + if (std::error_code EC = Cutoff.getError()) + return EC; + + auto MinBlockCount = readNumber<uint64_t>(); + if (std::error_code EC = MinBlockCount.getError()) + return EC; + + auto NumBlocks = readNumber<uint64_t>(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readSummary() { + auto TotalCount = readNumber<uint64_t>(); + if (std::error_code EC = TotalCount.getError()) + return EC; + + auto MaxBlockCount = readNumber<uint64_t>(); + if (std::error_code EC = MaxBlockCount.getError()) + return EC; + + auto MaxFunctionCount = readNumber<uint64_t>(); + if (std::error_code EC = MaxFunctionCount.getError()) + return EC; + + auto NumBlocks = readNumber<uint64_t>(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + auto NumFunctions = readNumber<uint64_t>(); + if (std::error_code EC = NumFunctions.getError()) + return EC; + + auto NumSummaryEntries = readNumber<uint64_t>(); + if (std::error_code EC = NumSummaryEntries.getError()) + return EC; + + std::vector<ProfileSummaryEntry> Entries; + for (unsigned i = 0; i < *NumSummaryEntries; i++) { + std::error_code EC = readSummaryEntry(Entries); + if (EC != sampleprof_error::success) + return EC; + } + Summary = llvm::make_unique<ProfileSummary>( + ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0, + *MaxFunctionCount, *NumBlocks, *NumFunctions); + + return sampleprof_error::success; +} + bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) { const uint8_t *Data = reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); @@ -518,6 +580,7 @@ std::error_code SampleProfileReaderGCC::readFunctionProfiles() { if (std::error_code EC = readOneFunctionProfile(Stack, true, 0)) return EC; + computeSummary(); return sampleprof_error::success; } @@ -562,8 +625,9 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile( uint32_t LineOffset = Offset >> 16; uint32_t Discriminator = Offset & 0xffff; FProfile = &CallerProfile->functionSamplesAt( - CallsiteLocation(LineOffset, Discriminator, Name)); + LineLocation(LineOffset, Discriminator)); } + FProfile->setName(Name); for (uint32_t I = 0; I < NumPosCounts; ++I) { uint32_t Offset; @@ -669,7 +733,7 @@ bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) { /// /// \returns an error code indicating the status of the buffer. static ErrorOr<std::unique_ptr<MemoryBuffer>> -setupMemoryBuffer(std::string Filename) { +setupMemoryBuffer(const Twine &Filename) { auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename); if (std::error_code EC = BufferOrErr.getError()) return EC; @@ -692,7 +756,7 @@ setupMemoryBuffer(std::string Filename) { /// /// \returns an error code indicating the status of the created reader. ErrorOr<std::unique_ptr<SampleProfileReader>> -SampleProfileReader::create(StringRef Filename, LLVMContext &C) { +SampleProfileReader::create(const Twine &Filename, LLVMContext &C) { auto BufferOrError = setupMemoryBuffer(Filename); if (std::error_code EC = BufferOrError.getError()) return EC; @@ -725,3 +789,14 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) { return std::move(Reader); } + +// For text and GCC file formats, we compute the summary after reading the +// profile. Binary format has the profile summary in its header. +void SampleProfileReader::computeSummary() { + SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); + for (const auto &I : Profiles) { + const FunctionSamples &Profile = I.second; + Builder.addRecord(Profile); + } + Summary = Builder.getSummary(); +} diff --git a/gnu/llvm/lib/ProfileData/SampleProfWriter.cpp b/gnu/llvm/lib/ProfileData/SampleProfWriter.cpp index 51feee5ad7d..4fa71288f8d 100644 --- a/gnu/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/gnu/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -37,11 +37,9 @@ using namespace llvm; /// /// The format used here is more structured and deliberate because /// it needs to be parsed by the SampleProfileReaderText class. -std::error_code SampleProfileWriterText::write(StringRef FName, - const FunctionSamples &S) { +std::error_code SampleProfileWriterText::write(const FunctionSamples &S) { auto &OS = *OutputStream; - - OS << FName << ":" << S.getTotalSamples(); + OS << S.getName() << ":" << S.getTotalSamples(); if (Indent == 0) OS << ":" << S.getHeadSamples(); OS << "\n"; @@ -63,18 +61,18 @@ std::error_code SampleProfileWriterText::write(StringRef FName, OS << "\n"; } - SampleSorter<CallsiteLocation, FunctionSamples> SortedCallsiteSamples( + SampleSorter<LineLocation, FunctionSamples> SortedCallsiteSamples( S.getCallsiteSamples()); Indent += 1; for (const auto &I : SortedCallsiteSamples.get()) { - CallsiteLocation Loc = I->first; + LineLocation Loc = I->first; const FunctionSamples &CalleeSamples = I->second; OS.indent(Indent); if (Loc.Discriminator == 0) OS << Loc.LineOffset << ": "; else OS << Loc.LineOffset << "." << Loc.Discriminator << ": "; - if (std::error_code EC = write(Loc.CalleeName, CalleeSamples)) + if (std::error_code EC = write(CalleeSamples)) return EC; } Indent -= 1; @@ -105,9 +103,8 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { // Recursively add all the names for inlined callsites. for (const auto &J : S.getCallsiteSamples()) { - CallsiteLocation Loc = J.first; const FunctionSamples &CalleeSamples = J.second; - addName(Loc.CalleeName); + addName(CalleeSamples.getName()); addNames(CalleeSamples); } } @@ -120,6 +117,10 @@ std::error_code SampleProfileWriterBinary::writeHeader( encodeULEB128(SPMagic(), OS); encodeULEB128(SPVersion(), OS); + computeSummary(ProfileMap); + if (auto EC = writeSummary()) + return EC; + // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { addName(I.first()); @@ -132,15 +133,29 @@ std::error_code SampleProfileWriterBinary::writeHeader( OS << N.first; encodeULEB128(0, OS); } - return sampleprof_error::success; } -std::error_code SampleProfileWriterBinary::writeBody(StringRef FName, - const FunctionSamples &S) { +std::error_code SampleProfileWriterBinary::writeSummary() { + auto &OS = *OutputStream; + encodeULEB128(Summary->getTotalCount(), OS); + encodeULEB128(Summary->getMaxCount(), OS); + encodeULEB128(Summary->getMaxFunctionCount(), OS); + encodeULEB128(Summary->getNumCounts(), OS); + encodeULEB128(Summary->getNumFunctions(), OS); + std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary(); + encodeULEB128(Entries.size(), OS); + for (auto Entry : Entries) { + encodeULEB128(Entry.Cutoff, OS); + encodeULEB128(Entry.MinCount, OS); + encodeULEB128(Entry.NumCounts, OS); + } + return sampleprof_error::success; +} +std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { auto &OS = *OutputStream; - if (std::error_code EC = writeNameIdx(FName)) + if (std::error_code EC = writeNameIdx(S.getName())) return EC; encodeULEB128(S.getTotalSamples(), OS); @@ -166,11 +181,11 @@ std::error_code SampleProfileWriterBinary::writeBody(StringRef FName, // Recursively emit all the callsite samples. encodeULEB128(S.getCallsiteSamples().size(), OS); for (const auto &J : S.getCallsiteSamples()) { - CallsiteLocation Loc = J.first; + LineLocation Loc = J.first; const FunctionSamples &CalleeSamples = J.second; encodeULEB128(Loc.LineOffset, OS); encodeULEB128(Loc.Discriminator, OS); - if (std::error_code EC = writeBody(Loc.CalleeName, CalleeSamples)) + if (std::error_code EC = writeBody(CalleeSamples)) return EC; } @@ -180,10 +195,9 @@ std::error_code SampleProfileWriterBinary::writeBody(StringRef FName, /// \brief Write samples of a top-level function to a binary file. /// /// \returns true if the samples were written successfully, false otherwise. -std::error_code SampleProfileWriterBinary::write(StringRef FName, - const FunctionSamples &S) { +std::error_code SampleProfileWriterBinary::write(const FunctionSamples &S) { encodeULEB128(S.getHeadSamples(), *OutputStream); - return writeBody(FName, S); + return writeBody(S); } /// \brief Create a sample profile file writer based on the specified format. @@ -238,3 +252,13 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, return std::move(Writer); } + +void SampleProfileWriter::computeSummary( + const StringMap<FunctionSamples> &ProfileMap) { + SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); + for (const auto &I : ProfileMap) { + const FunctionSamples &Profile = I.second; + Builder.addRecord(Profile); + } + Summary = Builder.getSummary(); +} |
