From 8c0b3787e7ccc7978b42dfbb84da2b802c743a5d Mon Sep 17 00:00:00 2001
From: David Blaikie <dblaikie@gmail.com>
Date: Wed, 6 Jun 2012 18:52:13 +0000
Subject: [PATCH] Add a -rewrite-includes option, which is similar to
-rewrite-macros, but only expands #include directives.
Patch contributed by Lubos Lunak (l.lunax@suse.cz).
Review by Matt Beaumont-Gay (matthewbg@google.com).
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@158093 91177308-0d34-0410-b5e6-96231b3b80d8
---
include/clang/Driver/CC1Options.td | 2 +
include/clang/Frontend/FrontendOptions.h | 1 +
include/clang/Lex/Preprocessor.h | 13 +
include/clang/Rewrite/FrontendActions.h | 5 +
include/clang/Rewrite/Rewriters.h | 5 +
lib/Frontend/CompilerInvocation.cpp | 3 +
.../ExecuteCompilerInvocation.cpp | 1 +
lib/Lex/Lexer.cpp | 5 +-
lib/Lex/PPDirectives.cpp | 19 +
lib/Lex/Preprocessor.cpp | 1 +
lib/Rewrite/CMakeLists.txt | 1 +
lib/Rewrite/FrontendActions.cpp | 9 +
lib/Rewrite/InclusionRewriter.cpp | 370 ++++++++++++++++++
test/Frontend/Inputs/rewrite-includes1.h | 3 +
test/Frontend/Inputs/rewrite-includes2.h | 1 +
test/Frontend/Inputs/rewrite-includes3.h | 1 +
test/Frontend/Inputs/rewrite-includes4.h | 1 +
test/Frontend/Inputs/rewrite-includes5.h | 1 +
test/Frontend/Inputs/rewrite-includes6.h | 2 +
test/Frontend/Inputs/rewrite-includes7.h | 4 +
test/Frontend/rewrite-includes.c | 145 +++++++
21 files changed, 591 insertions(+), 2 deletions(-)
create mode 100644 lib/Rewrite/InclusionRewriter.cpp
create mode 100644 test/Frontend/Inputs/rewrite-includes1.h
create mode 100644 test/Frontend/Inputs/rewrite-includes2.h
create mode 100644 test/Frontend/Inputs/rewrite-includes3.h
create mode 100644 test/Frontend/Inputs/rewrite-includes4.h
create mode 100644 test/Frontend/Inputs/rewrite-includes5.h
create mode 100644 test/Frontend/Inputs/rewrite-includes6.h
create mode 100644 test/Frontend/Inputs/rewrite-includes7.h
create mode 100644 test/Frontend/rewrite-includes.c
diff --git a/include/clang/Driver/CC1Options.td b/include/clang/Driver/CC1Options.td
index 74497de3cf1..b4aba8d31b6 100644
--- a/include/clang/Driver/CC1Options.td
+++ b/include/clang/Driver/CC1Options.td
@@ -332,6 +332,8 @@ def rewrite_test : Flag<"-rewrite-test">,
HelpText<"Rewriter playground">;
def rewrite_macros : Flag<"-rewrite-macros">,
HelpText<"Expand macros without full preprocessing">;
+def rewrite_includes : Flag<"-rewrite-includes">,
+ HelpText<"Expand includes without full preprocessing">;
def migrate : Flag<"-migrate">,
HelpText<"Migrate source code">;
}
diff --git a/include/clang/Frontend/FrontendOptions.h b/include/clang/Frontend/FrontendOptions.h
index 888388c310e..78d3adfc6bf 100644
--- a/include/clang/Frontend/FrontendOptions.h
+++ b/include/clang/Frontend/FrontendOptions.h
@@ -43,6 +43,7 @@ namespace frontend {
PrintPreamble, ///< Print the "preamble" of the input file
PrintPreprocessedInput, ///< -E mode.
RewriteMacros, ///< Expand macros but not #includes.
+ RewriteIncludes, ///< Expand #includes but not macros.
RewriteObjC, ///< ObjC->C Rewriter.
RewriteTest, ///< Rewriter playground
RunAnalysis, ///< Run one or more source code analyses.
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index 737a1526505..6589f84c931 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -121,6 +121,13 @@ class Preprocessor : public RefCountedBase<Preprocessor> {
/// DisableMacroExpansion - True if macro expansion is disabled.
bool DisableMacroExpansion : 1;
+ /// MacroExpansionInDirectivesOverride - Temporarily disables
+ /// DisableMacroExpansion (i.e. enables expansion) when parsing preprocessor
+ /// directives.
+ bool MacroExpansionInDirectivesOverride : 1;
+
+ class ResetMacroExpansionHelper;
+
/// \brief Whether we have already loaded macros from the external source.
mutable bool ReadMacrosFromExternalSource : 1;
@@ -643,6 +650,12 @@ public:
while (Result.getKind() == tok::comment);
}
+ /// Disables macro expansion everywhere except for preprocessor directives.
+ void SetMacroExpansionOnlyInDirectives() {
+ DisableMacroExpansion = true;
+ MacroExpansionInDirectivesOverride = true;
+ }
+
/// LookAhead - This peeks ahead N tokens and returns that token without
/// consuming any tokens. LookAhead(0) returns the next token that would be
/// returned by Lex(), LookAhead(1) returns the token after it, etc. This
diff --git a/include/clang/Rewrite/FrontendActions.h b/include/clang/Rewrite/FrontendActions.h
index 6e9ecace672..ea876d9980f 100644
--- a/include/clang/Rewrite/FrontendActions.h
+++ b/include/clang/Rewrite/FrontendActions.h
@@ -73,6 +73,11 @@ protected:
void ExecuteAction();
};
+class RewriteIncludesAction : public PreprocessorFrontendAction {
+protected:
+ void ExecuteAction();
+};
+
} // end namespace clang
#endif
diff --git a/include/clang/Rewrite/Rewriters.h b/include/clang/Rewrite/Rewriters.h
index 203b9bc18b3..4283fda2f36 100644
--- a/include/clang/Rewrite/Rewriters.h
+++ b/include/clang/Rewrite/Rewriters.h
@@ -18,6 +18,7 @@
namespace clang {
class Preprocessor;
+class PreprocessorOutputOptions;
/// RewriteMacrosInInput - Implement -rewrite-macros mode.
void RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS);
@@ -25,6 +26,10 @@ void RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS);
/// DoRewriteTest - A simple test for the TokenRewriter class.
void DoRewriteTest(Preprocessor &PP, raw_ostream *OS);
+/// RewriteIncludesInInput - Implement -rewrite-includes mode.
+void RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
+ const PreprocessorOutputOptions &Opts);
+
} // end namespace clang
#endif
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index 5a3732bf437..35ffe4941cb 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -445,6 +445,7 @@ static const char *getActionName(frontend::ActionKind Kind) {
case frontend::PrintPreamble: return "-print-preamble";
case frontend::PrintPreprocessedInput: return "-E";
case frontend::RewriteMacros: return "-rewrite-macros";
+ case frontend::RewriteIncludes: return "-rewrite-includes";
case frontend::RewriteObjC: return "-rewrite-objc";
case frontend::RewriteTest: return "-rewrite-test";
case frontend::RunAnalysis: return "-analyze";
@@ -1435,6 +1436,8 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
Opts.ProgramAction = frontend::PrintPreprocessedInput; break;
case OPT_rewrite_macros:
Opts.ProgramAction = frontend::RewriteMacros; break;
+ case OPT_rewrite_includes:
+ Opts.ProgramAction = frontend::RewriteIncludes; break;
case OPT_rewrite_objc:
Opts.ProgramAction = frontend::RewriteObjC; break;
case OPT_rewrite_test:
diff --git a/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index f72d294e7ae..d4e47bac4e2 100644
--- a/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -73,6 +73,7 @@ static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
case PrintPreamble: return new PrintPreambleAction();
case PrintPreprocessedInput: return new PrintPreprocessedAction();
case RewriteMacros: return new RewriteMacrosAction();
+ case RewriteIncludes: return new RewriteIncludesAction();
case RewriteObjC: return new RewriteObjCAction();
case RewriteTest: return new RewriteTestAction();
case RunAnalysis: return new ento::AnalysisAction();
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index a43543ad9a4..4ab0dae1fa1 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -2022,7 +2022,7 @@ bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
// directly.
FormTokenWithChars(Result, CurPtr, tok::comment);
- if (!ParsingPreprocessorDirective)
+ if (!ParsingPreprocessorDirective || LexingRawMode)
return true;
// If this BCPL-style comment is in a macro definition, transmogrify it into
@@ -2626,7 +2626,8 @@ LexNextToken:
ParsingPreprocessorDirective = false;
// Restore comment saving mode, in case it was disabled for directive.
- SetCommentRetentionState(PP->getCommentRetentionState());
+ if (!LexingRawMode)
+ SetCommentRetentionState(PP->getCommentRetentionState());
// Since we consumed a newline, we are back at the start of a line.
IsAtStartOfLine = true;
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index c6290170892..5c03c97b7c3 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -553,6 +553,21 @@ const FileEntry *Preprocessor::LookupFile(
// Preprocessor Directive Handling.
//===----------------------------------------------------------------------===//
+class Preprocessor::ResetMacroExpansionHelper {
+public:
+ ResetMacroExpansionHelper(Preprocessor *pp)
+ : PP(pp), save(pp->DisableMacroExpansion) {
+ if (pp->MacroExpansionInDirectivesOverride)
+ pp->DisableMacroExpansion = false;
+ }
+ ~ResetMacroExpansionHelper() {
+ PP->DisableMacroExpansion = save;
+ }
+private:
+ Preprocessor *PP;
+ bool save;
+};
+
/// HandleDirective - This callback is invoked when the lexer sees a # token
/// at the start of a line. This consumes the directive, modifies the
/// lexer/preprocessor state, and advances the lexer(s) so that the next token
@@ -604,6 +619,10 @@ void Preprocessor::HandleDirective(Token &Result) {
Diag(Result, diag::ext_embedded_directive);
}
+ // Temporarily enable macro expansion if set so
+ // and reset to previous state when returning from this function.
+ ResetMacroExpansionHelper helper(this);
+
TryAgain:
switch (Result.getKind()) {
case tok::eod:
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 955c39cfe78..4be59c65f77 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -86,6 +86,7 @@ Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
// Macro expansion is enabled.
DisableMacroExpansion = false;
+ MacroExpansionInDirectivesOverride = false;
InMacroArgs = false;
InMacroArgPreExpansion = false;
NumCachedTokenLexers = 0;
diff --git a/lib/Rewrite/CMakeLists.txt b/lib/Rewrite/CMakeLists.txt
index 2a050401938..091af9977f5 100644
--- a/lib/Rewrite/CMakeLists.txt
+++ b/lib/Rewrite/CMakeLists.txt
@@ -6,6 +6,7 @@ add_clang_library(clangRewrite
FrontendActions.cpp
HTMLPrint.cpp
HTMLRewrite.cpp
+ InclusionRewriter.cpp
RewriteMacros.cpp
RewriteModernObjC.cpp
RewriteObjC.cpp
diff --git a/lib/Rewrite/FrontendActions.cpp b/lib/Rewrite/FrontendActions.cpp
index 1753325a26c..fa58575258e 100644
--- a/lib/Rewrite/FrontendActions.cpp
+++ b/lib/Rewrite/FrontendActions.cpp
@@ -181,3 +181,12 @@ void RewriteTestAction::ExecuteAction() {
DoRewriteTest(CI.getPreprocessor(), OS);
}
+
+void RewriteIncludesAction::ExecuteAction() {
+ CompilerInstance &CI = getCompilerInstance();
+ raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
+ if (!OS) return;
+
+ RewriteIncludesInInput(CI.getPreprocessor(), OS,
+ CI.getPreprocessorOutputOpts());
+}
diff --git a/lib/Rewrite/InclusionRewriter.cpp b/lib/Rewrite/InclusionRewriter.cpp
new file mode 100644
index 00000000000..439d777f6c8
--- /dev/null
+++ b/lib/Rewrite/InclusionRewriter.cpp
@@ -0,0 +1,370 @@
+//===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This code rewrites include invocations into their expansions. This gives you
+// a file with all included files merged into it.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Rewriters.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/PreprocessorOutputOptions.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+using namespace llvm;
+
+namespace {
+
+class InclusionRewriter : public PPCallbacks {
+ /// Information about which #includes were actually performed,
+ /// created by preprocessor callbacks.
+ struct FileChange {
+ SourceLocation From;
+ FileID Id;
+ SrcMgr::CharacteristicKind FileType;
+ FileChange(SourceLocation From) : From(From) {
+ }
+ };
+ Preprocessor &PP; //< Used to find inclusion directives.
+ SourceManager &SM; //< Used to read and manage source files.
+ raw_ostream &OS; //< The destination stream for rewritten contents.
+ bool ShowLineMarkers; //< Show #line markers.
+ bool UseLineDirective; //< Use of line directives or line markers.
+ typedef std::map<unsigned, FileChange> FileChangeMap;
+ FileChangeMap FileChanges; /// Tracks which files were included where.
+ /// Used transitively for building up the FileChanges mapping over the
+ /// various \c PPCallbacks callbacks.
+ FileChangeMap::iterator LastInsertedFileChange;
+public:
+ InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers);
+ bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
+private:
+ virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
+ SrcMgr::CharacteristicKind FileType,
+ FileID PrevFID);
+ virtual void FileSkipped(const FileEntry &ParentFile,
+ const Token &FilenameTok,
+ SrcMgr::CharacteristicKind FileType);
+ virtual void InclusionDirective(SourceLocation HashLoc,
+ const Token &IncludeTok,
+ StringRef FileName,
+ bool IsAngled,
+ const FileEntry *File,
+ SourceLocation EndLoc,
+ StringRef SearchPath,
+ StringRef RelativePath);
+ void WriteLineInfo(const char *Filename, int Line,
+ SrcMgr::CharacteristicKind FileType,
+ StringRef EOL, StringRef Extra = StringRef());
+ void OutputContentUpTo(const MemoryBuffer &FromFile,
+ unsigned &WriteFrom, unsigned WriteTo,
+ StringRef EOL, int &lines,
+ bool EnsureNewline = false);
+ void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
+ const MemoryBuffer &FromFile, StringRef EOL,
+ unsigned &NextToWrite, int &Lines);
+ const FileChange *FindFileChangeLocation(SourceLocation Loc) const;
+ StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
+};
+
+} // end anonymous namespace
+
+/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
+InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
+ bool ShowLineMarkers)
+ : PP(PP), SM(PP.getSourceManager()), OS(OS),
+ ShowLineMarkers(ShowLineMarkers),
+ LastInsertedFileChange(FileChanges.end()) {
+ // If we're in microsoft mode, use normal #line instead of line markers.
+ UseLineDirective = PP.getLangOpts().MicrosoftExt;
+}
+
+/// Write appropriate line information as either #line directives or GNU line
+/// markers depending on what mode we're in, including the \p Filename and
+/// \p Line we are located at, using the specified \p EOL line separator, and
+/// any \p Extra context specifiers in GNU line directives.
+void InclusionRewriter::WriteLineInfo(const char *Filename, int Line,
+ SrcMgr::CharacteristicKind FileType,
+ StringRef EOL, StringRef Extra) {
+ if (!ShowLineMarkers)
+ return;
+ if (UseLineDirective) {
+ OS << "#line" << ' ' << Line << ' ' << '"' << Filename << '"';
+ } else {
+ // Use GNU linemarkers as described here:
+ // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
+ OS << '#' << ' ' << Line << ' ' << '"' << Filename << '"';
+ if (!Extra.empty())
+ OS << Extra;
+ if (FileType == SrcMgr::C_System)
+ // "`3' This indicates that the following text comes from a system header
+ // file, so certain warnings should be suppressed."
+ OS << " 3";
+ else if (FileType == SrcMgr::C_ExternCSystem)
+ // as above for `3', plus "`4' This indicates that the following text
+ // should be treated as being wrapped in an implicit extern "C" block."
+ OS << " 3 4";
+ }
+ OS << EOL;
+}
+
+/// FileChanged - Whenever the preprocessor enters or exits a #include file
+/// it invokes this handler.
+void InclusionRewriter::FileChanged(SourceLocation Loc,
+ FileChangeReason Reason,
+ SrcMgr::CharacteristicKind NewFileType,
+ FileID) {
+ if (Reason != EnterFile)
+ return;
+ if (LastInsertedFileChange == FileChanges.end())
+ // we didn't reach this file (eg: the main file) via an inclusion directive
+ return;
+ LastInsertedFileChange->second.Id = FullSourceLoc(Loc, SM).getFileID();
+ LastInsertedFileChange->second.FileType = NewFileType;
+ LastInsertedFileChange = FileChanges.end();
+}
+
+/// Called whenever an inclusion is skipped due to canonical header protection
+/// macros.
+void InclusionRewriter::FileSkipped(const FileEntry &/*ParentFile*/,
+ const Token &/*FilenameTok*/,
+ SrcMgr::CharacteristicKind /*FileType*/) {
+ assert(LastInsertedFileChange != FileChanges.end() && "A file, that wasn't "
+ "found via an inclusion directive, was skipped");
+ FileChanges.erase(LastInsertedFileChange);
+ LastInsertedFileChange = FileChanges.end();
+}
+
+/// This should be called whenever the preprocessor encounters include
+/// directives. It does not say whether the file has been included, but it
+/// provides more information about the directive (hash location instead
+/// of location inside the included file). It is assumed that the matching
+/// FileChanged() or FileSkipped() is called after this.
+void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
+ const Token &/*IncludeTok*/,
+ StringRef /*FileName*/,
+ bool /*IsAngled*/,
+ const FileEntry * /*File*/,
+ SourceLocation /*EndLoc*/,
+ StringRef /*SearchPath*/,
+ StringRef /*RelativePath*/) {
+ assert(LastInsertedFileChange == FileChanges.end() && "Another inclusion "
+ "directive was found before the previous one was processed");
+ std::pair<FileChangeMap::iterator, bool> p = FileChanges.insert(
+ std::make_pair(HashLoc.getRawEncoding(), FileChange(HashLoc)));
+ assert(p.second && "Unexpected revisitation of the same include directive");
+ LastInsertedFileChange = p.first;
+}
+
+/// Simple lookup for a SourceLocation (specifically one denoting the hash in
+/// an inclusion directive) in the map of inclusion information, FileChanges.
+const InclusionRewriter::FileChange *
+InclusionRewriter::FindFileChangeLocation(SourceLocation Loc) const {
+ FileChangeMap::const_iterator I = FileChanges.find(Loc.getRawEncoding());
+ if (I != FileChanges.end())
+ return &I->second;
+ return NULL;
+}
+
+/// Count the raw \\n characters in the \p Len characters from \p Pos.
+inline unsigned CountNewLines(const char *Pos, int Len) {
+ const char *End = Pos + Len;
+ unsigned Lines = 0;
+ --Pos;
+ while ((Pos = static_cast<const char*>(memchr(Pos + 1, '\n', End - Pos - 1))))
+ ++Lines;
+ return Lines;
+}
+
+/// Detect the likely line ending style of \p FromFile by examining the first
+/// newline found within it.
+static StringRef DetectEOL(const MemoryBuffer &FromFile) {
+ // detect what line endings the file uses, so that added content does not mix
+ // the style
+ const char *Pos = strchr(FromFile.getBufferStart(), '\n');
+ if (Pos == NULL)
+ return "\n";
+ if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
+ return "\n\r";
+ if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
+ return "\r\n";
+ return "\n";
+}
+
+/// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
+/// \p WriteTo - 1.
+void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
+ unsigned &WriteFrom, unsigned WriteTo,
+ StringRef EOL, int &Line,
+ bool EnsureNewline) {
+ if (WriteTo <= WriteFrom)
+ return;
+ OS.write(FromFile.getBufferStart() + WriteFrom, WriteTo - WriteFrom);
+ // count lines manually, it's faster than getPresumedLoc()
+ Line += CountNewLines(FromFile.getBufferStart() + WriteFrom,
+ WriteTo - WriteFrom);
+ if (EnsureNewline) {
+ char LastChar = FromFile.getBufferStart()[WriteTo - 1];
+ if (LastChar != '\n' && LastChar != '\r')
+ OS << EOL;
+ }
+ WriteFrom = WriteTo;
+}
+
+/// Print characters from \p FromFile starting at \p NextToWrite up until the
+/// inclusion directive at \p StartToken, then print out the inclusion
+/// inclusion directive disabled by a #if directive, updating \p NextToWrite
+/// and \p Line to track the number of source lines visited and the progress
+/// through the \p FromFile buffer.
+void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
+ const Token &StartToken,
+ const MemoryBuffer &FromFile,
+ StringRef EOL,
+ unsigned &NextToWrite, int &Line) {
+ OutputContentUpTo(FromFile, NextToWrite,
+ SM.getFileOffset(StartToken.getLocation()), EOL, Line);
+ Token DirectiveToken;
+ do {
+ DirectiveLex.LexFromRawLexer(DirectiveToken);
+ } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
+ OS << "#if 0 /* expanded by -rewrite-includes */" << EOL;
+ OutputContentUpTo(FromFile, NextToWrite,
+ SM.getFileOffset(DirectiveToken.getLocation()) + DirectiveToken.getLength(),
+ EOL, Line);
+ OS << "#endif /* expanded by -rewrite-includes */" << EOL;
+}
+
+/// Find the next identifier in the pragma directive specified by \p RawToken.
+StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
+ Token &RawToken) {
+ RawLex.LexFromRawLexer(RawToken);
+ if (RawToken.is(tok::raw_identifier))
+ PP.LookUpIdentifierInfo(RawToken);
+ if (RawToken.is(tok::identifier))
+ return RawToken.getIdentifierInfo()->getName();
+ return StringRef();
+}
+
+/// Use a raw lexer to analyze \p FileId, inccrementally copying parts of it
+/// and including content of included files recursively.
+bool InclusionRewriter::Process(FileID FileId,
+ SrcMgr::CharacteristicKind FileType)
+{
+ bool Invalid;
+ const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
+ assert(!Invalid && "Invalid FileID while trying to rewrite includes");
+ const char *FileName = FromFile.getBufferIdentifier();
+ Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
+ RawLex.SetCommentRetentionState(false);
+
+ StringRef EOL = DetectEOL(FromFile);
+
+ // Per the GNU docs: "1" indicates the start of a new file.
+ WriteLineInfo(FileName, 1, FileType, EOL, " 1");
+
+ if (SM.getFileIDSize(FileId) == 0)
+ return true;
+
+ // The next byte to be copied from the source file
+ unsigned NextToWrite = 0;
+ int Line = 1; // The current input file line number.
+
+ Token RawToken;
+ RawLex.LexFromRawLexer(RawToken);
+
+ // TODO: Consider adding a switch that strips possibly unimportant content,
+ // such as comments, to reduce the size of repro files.
+ while (RawToken.isNot(tok::eof)) {
+ if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
+ RawLex.setParsingPreprocessorDirective(true);
+ Token HashToken = RawToken;
+ RawLex.LexFromRawLexer(RawToken);
+ if (RawToken.is(tok::raw_identifier))
+ PP.LookUpIdentifierInfo(RawToken);
+ if (RawToken.is(tok::identifier)) {
+ switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
+ case tok::pp_include:
+ case tok::pp_include_next:
+ case tok::pp_import: {
+ CommentOutDirective(RawLex, HashToken, FromFile, EOL, NextToWrite,
+ Line);
+ if (const FileChange *Change = FindFileChangeLocation(
+ HashToken.getLocation())) {
+ // now include and recursively process the file
+ if (Process(Change->Id, Change->FileType))
+ // and set lineinfo back to this file, if the nested one was
+ // actually included
+ // `2' indicates returning to a file (after having included
+ // another file.
+ WriteLineInfo(FileName, Line, FileType, EOL, " 2");
+ } else
+ // fix up lineinfo (since commented out directive changed line
+ // numbers) for inclusions that were skipped due to header guards
+ WriteLineInfo(FileName, Line, FileType, EOL);
+ break;
+ }
+ case tok::pp_pragma: {
+ StringRef Identifier = NextIdentifierName(RawLex, RawToken);
+ if (Identifier == "clang" || Identifier == "GCC") {
+ if (NextIdentifierName(RawLex, RawToken) == "system_header") {
+ // keep the directive in, commented out
+ CommentOutDirective(RawLex, HashToken, FromFile, EOL,
+ NextToWrite, Line);
+ // update our own type
+ FileType = SM.getFileCharacteristic(RawToken.getLocation());
+ WriteLineInfo(FileName, Line, FileType, EOL);
+ }
+ } else if (Identifier == "once") {
+ // keep the directive in, commented out
+ CommentOutDirective(RawLex, HashToken, FromFile, EOL,
+ NextToWrite, Line);
+ WriteLineInfo(FileName, Line, FileType, EOL);
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ RawLex.setParsingPreprocessorDirective(false);
+ }
+ RawLex.LexFromRawLexer(RawToken);
+ }
+ OutputContentUpTo(FromFile, NextToWrite,
+ SM.getFileOffset(SM.getLocForEndOfFile(FileId)) + 1, EOL, Line,
+ /*EnsureNewline*/true);
+ return true;
+}
+
+/// InclusionRewriterInInput - Implement -rewrite-includes mode.
+void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
+ const PreprocessorOutputOptions &Opts) {
+ SourceManager &SM = PP.getSourceManager();
+ InclusionRewriter *Rewrite = new InclusionRewriter(PP, *OS,
+ Opts.ShowLineMarkers);
+ PP.addPPCallbacks(Rewrite);
+
+ // First let the preprocessor process the entire file and call callbacks.
+ // Callbacks will record which #include's were actually performed.
+ PP.EnterMainSourceFile();
+ Token Tok;
+ // Only preprocessor directives matter here, so disable macro expansion
+ // everywhere else as an optimization.
+ // TODO: It would be even faster if the preprocessor could be switched
+ // to a mode where it would parse only preprocessor directives and comments,
+ // nothing else matters for parsing or processing.
+ PP.SetMacroExpansionOnlyInDirectives();
+ do {
+ PP.Lex(Tok);
+ } while (Tok.isNot(tok::eof));
+ Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
+ OS->flush();
+}
diff --git a/test/Frontend/Inputs/rewrite-includes1.h b/test/Frontend/Inputs/rewrite-includes1.h
new file mode 100644
index 00000000000..1b6c80d5f43
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes1.h
@@ -0,0 +1,3 @@
+#pragma clang system_header
+included_line1
+#include "rewrite-includes2.h"
diff --git a/test/Frontend/Inputs/rewrite-includes2.h b/test/Frontend/Inputs/rewrite-includes2.h
new file mode 100644
index 00000000000..1114e51cc13
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes2.h
@@ -0,0 +1 @@
+included_line2
diff --git a/test/Frontend/Inputs/rewrite-includes3.h b/test/Frontend/Inputs/rewrite-includes3.h
new file mode 100644
index 00000000000..3757bc8b341
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes3.h
@@ -0,0 +1 @@
+included_line3
diff --git a/test/Frontend/Inputs/rewrite-includes4.h b/test/Frontend/Inputs/rewrite-includes4.h
new file mode 100644
index 00000000000..b4e25d26f62
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes4.h
@@ -0,0 +1 @@
+included_line4
diff --git a/test/Frontend/Inputs/rewrite-includes5.h b/test/Frontend/Inputs/rewrite-includes5.h
new file mode 100644
index 00000000000..934bf413e50
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes5.h
@@ -0,0 +1 @@
+included_line5
diff --git a/test/Frontend/Inputs/rewrite-includes6.h b/test/Frontend/Inputs/rewrite-includes6.h
new file mode 100644
index 00000000000..c18e5013aff
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes6.h
@@ -0,0 +1,2 @@
+#pragma once
+included_line6
diff --git a/test/Frontend/Inputs/rewrite-includes7.h b/test/Frontend/Inputs/rewrite-includes7.h
new file mode 100644
index 00000000000..da00d4fb97d
--- /dev/null
+++ b/test/Frontend/Inputs/rewrite-includes7.h
@@ -0,0 +1,4 @@
+#ifndef REWRITE_INCLUDES_7
+#define REWRITE_INCLUDES_7
+included_line7
+#endif
diff --git a/test/Frontend/rewrite-includes.c b/test/Frontend/rewrite-includes.c
new file mode 100644
index 00000000000..58d82ab407f
--- /dev/null
+++ b/test/Frontend/rewrite-includes.c
@@ -0,0 +1,145 @@
+// RUN: %clang_cc1 -verify -rewrite-includes -DFIRST -I %S/Inputs %s -o - | FileCheck -strict-whitespace %s
+// RUN: %clang_cc1 -verify -rewrite-includes -P -DFIRST -I %S/Inputs %s -o - | FileCheck -check-prefix=CHECKNL -strict-whitespace %s
+// STARTCOMPARE
+#define A(a,b) a ## b
+A(1,2)
+#include "rewrite-includes1.h"
+#ifdef FIRST
+#define HEADER "rewrite-includes3.h"
+#include HEADER
+#else
+#include "rewrite-includes4.h"
+#endif
+#/**/include /**/ "rewrite-includes5.h" /**/ \
+
+#include "rewrite-includes6.h" // comment
+
+#include "rewrite-includes6.h" /* comment
+ continues */
+#include "rewrite-includes7.h"
+#include "rewrite-includes7.h"
+// ENDCOMPARE
+// CHECK: {{^}}// STARTCOMPARE{{$}}
+// CHECK-NEXT: {{^}}#define A(a,b) a ## b{{$}}
+// CHECK-NEXT: {{^}}A(1,2){{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes1.h"{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes1.h" 1{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#pragma clang system_header{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 2 "{{.*}}/Inputs/rewrite-includes1.h" 3{{$}}
+// CHECK-NEXT: {{^}}included_line1{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes2.h"{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes2.h" 1 3{{$}}
+// CHECK-NEXT: {{^}}included_line2{{$}}
+// CHECK-NEXT: {{^}}# 4 "{{.*}}/Inputs/rewrite-includes1.h" 2 3{{$}}
+// CHECK-NEXT: {{^}}# 7 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}#ifdef FIRST{{$}}
+// CHECK-NEXT: {{^}}#define HEADER "rewrite-includes3.h"{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include HEADER{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes3.h" 1{{$}}
+// CHECK-NEXT: {{^}}included_line3{{$}}
+// CHECK-NEXT: {{^}}# 10 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}#else{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes4.h"{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 12 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#/**/include /**/ "rewrite-includes5.h" /**/ {{\\}}{{$}}
+// CHECK-NEXT: {{^}} {{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes5.h" 1{{$}}
+// CHECK-NEXT: {{^}}included_line5{{$}}
+// CHECK-NEXT: {{^}}# 15 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes6.h" // comment{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes6.h" 1{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#pragma once{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 2 "{{.*}}/Inputs/rewrite-includes6.h"{{$}}
+// CHECK-NEXT: {{^}}included_line6{{$}}
+// CHECK-NEXT: {{^}}# 16 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}} {{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes6.h" /* comment{{$}}
+// CHECK-NEXT: {{^}} continues */{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 19 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes7.h" 1{{$}}
+// CHECK-NEXT: {{^}}#ifndef REWRITE_INCLUDES_7{{$}}
+// CHECK-NEXT: {{^}}#define REWRITE_INCLUDES_7{{$}}
+// CHECK-NEXT: {{^}}included_line7{{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}# 20 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}// ENDCOMPARE{{$}}
+
+// CHECKNL: {{^}}// STARTCOMPARE{{$}}
+// CHECKNL-NEXT: {{^}}#define A(a,b) a ## b{{$}}
+// CHECKNL-NEXT: {{^}}A(1,2){{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes1.h"{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#pragma clang system_header{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}included_line1{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes2.h"{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}included_line2{{$}}
+// CHECKNL-NEXT: {{^}}#ifdef FIRST{{$}}
+// CHECKNL-NEXT: {{^}}#define HEADER "rewrite-includes3.h"{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include HEADER{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}included_line3{{$}}
+// CHECKNL-NEXT: {{^}}#else{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes4.h"{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#/**/include /**/ "rewrite-includes5.h" /**/ {{\\}}{{$}}
+// CHECKNL-NEXT: {{^}} {{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}included_line5{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes6.h" // comment{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#pragma once{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}included_line6{{$}}
+// CHECKNL-NEXT: {{^}} {{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes6.h" /* comment{{$}}
+// CHECKNL-NEXT: {{^}} continues */{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#ifndef REWRITE_INCLUDES_7{{$}}
+// CHECKNL-NEXT: {{^}}#define REWRITE_INCLUDES_7{{$}}
+// CHECKNL-NEXT: {{^}}included_line7{{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}// ENDCOMPARE{{$}}
--
GitLab