Update cppbnf.cpp

7 files changed, 185 insertions, 90 deletions

diff --git a/Makefile b/Makefile
index a600db3..dbc8d2b 100644
--- a/Makefile
+++ b/Makefile
@@ -89,7 +89,7 @@ TESTSRC=\
 SRC=$(PROGSRC) mcc.cpp
 
 all: test-$(PROJECTNAME) mcc
-	./test-$(PROJECTNAME) # --gtest_filter='*preprocessing_tokenize*'
+	./test-$(PROJECTNAME) #--gtest_filter='*preprocessing_tokenize'
 
 # testsuite ----------------------------------------------
 test-$(PROJECTNAME): $(TESTSRC:.cpp=.o)
diff --git a/TODO b/TODO
index faafda7..f563810 100644
--- a/TODO
+++ b/TODO
@@ -1 +1,2 @@
-Update cppbnf.cpp to n4860
+Support/test empty compile unit
+grammer.cpp: match() : return point of match error ("Compile error")
diff --git a/cpp.cpp b/cpp.cpp
index 563ba4c..21f4df1 100644
--- a/cpp.cpp
+++ b/cpp.cpp
@@ -20,16 +20,7 @@ using namespace Gram;
 
 namespace fs = std::filesystem;
 
-CPP::CPP(): map_translation_unit ({
- {"/translation-unit/top-level-declaration-seq/top-level-declaration/declaration/function-definition",
-  [&](fs::path& path, index_t node_id)
-   {
-    //std::cout << "DEBUG: " << path << ", " << node_id << ", " << valueOfNode(node_id, m_nodes) << ", " << m_nodes[node_id].node_id << ", " << m_nodes[node_id].pos.node_id << std::endl;
-   }
- },
-})
-{
-}
+CPP::CPP(){}
 
 CPP::~CPP(){}
 
@@ -245,7 +236,8 @@ std::vector<Gram::TreeNode> CPP::analysis(const std::vector<Token>& tokens)
  return compiler.compile(tokens);
 }
 
-void CPP::traverse(index_t node_id, map_type& map, fs::path parent_path)
+#if 0
+void CPP::traverse(index_t node_id)
 {
  fs::path current_path{parent_path / m_nodes[node_id].type};
  
@@ -262,6 +254,11 @@ void CPP::traverse(index_t node_id, map_type& map, fs::path parent_path)
   }
  }
 }
+#endif
+
+void CPP::trTranslationUnit(index_t node_id)
+{
+}
 
 // Phase 7.c: Translate
 void CPP::translate()
@@ -269,7 +266,7 @@ void CPP::translate()
  if (m_nodes.size() == 0)
   throw std::runtime_error("ICE: Tree is empty");
 
- traverse(0, map_translation_unit);
+ trTranslationUnit(0);
 }
 
 // Phase 8: Instantiate objects
diff --git a/cpp.h b/cpp.h
index 6db7146..3be03e2 100644
--- a/cpp.h
+++ b/cpp.h
@@ -5,6 +5,13 @@
 
 #include <vector>
 
+struct CPPContext {
+ // global variable declarations
+ // global variable definitions
+ // functions declarations
+ // functions definitions
+};
+
 class CPP {
 
 public:
@@ -34,14 +41,12 @@ public:
  std::vector<uint8_t> getData();
 
 private:
- typedef std::unordered_map<std::string, std::function<void(fs::path&, index_t)>> map_type;
-
  std::string m_code; // input from compile()
  std::vector<Token> m_tokens; // result of phase 7.a
  std::vector<Gram::TreeNode> m_nodes; // result of phase 7.b
 
- void traverse(index_t node_id, map_type& map, fs::path parent_path = "/");
- 
- CPP::map_type map_translation_unit;
+ CPPContext m_cpp_context;
+
+ void trTranslationUnit(index_t node_id);
 };
 
diff --git a/cppbnf.cpp b/cppbnf.cpp
index d5ffa12..7e8ec15 100644
--- a/cppbnf.cpp
+++ b/cppbnf.cpp
@@ -199,13 +199,15 @@ BNF GetCppBNFLex()
   {"hex-quad", {{"hexadecimal-digit", "hexadecimal-digit", "hexadecimal-digit", "hexadecimal-digit"}}},
 
   {"universal-character-name", {
-   {"\\", "u", "hex-quad"},
-   {"\\", "U", "hex-quad", "hex-quad"}
+   {"\\u", "hex-quad"},
+   {"\\U", "hex-quad", "hex-quad"}
   }},
 
   {"preprocessing-token", {
    {"header-name"},
-   //{"import-keyword"}, // TODO
+   //{"import-keyword"}, // TODO: generated
+   //{"module-keyword"}, // TODO
+   //{"export-keyword"}, // TODO
    {"identifier"},
    {"pp-number"},
    {"character-literal"},
@@ -220,8 +222,7 @@ BNF GetCppBNFLex()
    {"identifier"},
    {"keyword"},
    {"literal"},
-   {"OPERATOR"}, // conflict with keyword "operator"
-   {"punctuator"},
+   {"operator-or-punctuator"},
   }},
 
   {"header-name", {
@@ -268,10 +269,97 @@ BNF GetCppBNFLex()
 
   {"digit", {{"0"},{"1"},{"2"},{"3"},{"4"},{"5"},{"6"},{"7"},{"8"},{"9"}}},
 
+  {"keyword", {
+   {"alignas"},
+   {"alignof"},
+   {"asm"},
+   {"auto"},
+   {"bool"},
+   {"break"},
+   {"case"},
+   {"catch"},
+   {"char"},
+   {"char8_t"},
+   {"char16_t"},
+   {"char32_t"},
+   {"class"},
+   {"concept"},
+   {"const"},
+   {"consteval"},
+   {"constexpr"},
+   {"constinit"},
+   {"const_cast"},
+   {"continue"},
+   {"co_await"},
+   {"co_return"},
+   {"co_yield"},
+   {"decltype"},
+   {"default"},
+   {"delete"},
+   {"do"},
+   {"double"},
+   {"dynamic_cast"},
+   {"else"},
+   {"enum"},
+   {"explicit"},
+   {"export"},
+   {"extern"},
+   {"false"},
+   {"float"},
+   {"for"},
+   {"friend"},
+   {"goto"},
+   {"if"},
+   {"inline"},
+   {"int"},
+   {"long"},
+   {"mutable"},
+   {"namespace"},
+   {"new"},
+   {"noexcept"},
+   {"nullptr"},
+   {"operator"},
+   {"private"},
+   {"protected"},
+   {"public"},
+   {"register"},
+   {"reinterpret_cast"},
+   {"requires"},
+   {"return"},
+   {"short"},
+   {"signed"},
+   {"sizeof"},
+   {"static"},
+   {"static_assert"},
+   {"static_cast"},
+   {"struct"},
+   {"switch"},
+   {"template"},
+   {"this"},
+   {"thread_local"},
+   {"throw"},
+   {"true"},
+   {"try"},
+   {"typedef"},
+   {"typeid"},
+   {"typename"},
+   {"union"},
+   {"unsigned"},
+   {"using"},
+   {"virtual"},
+   {"void"},
+   {"volatile"},
+   {"wchar_t"},
+   {"while"},
+   //{"import-keyword"}, // TODO
+   //{"module-keyword"},
+   //{"export-keyword"},
+  }},
+
   {"preprocessing-op-or-punc", {
-   {"{"}, {"}"}, {"["}, {"]"}, {"#"}, {"#", "#"}, {"("}, {")"},
-   {"<:"}, {":>"}, {"<%"}, {"%>"}, {"%:"}, {"%:%:"}, {";"}, {":"}, {"..."},
-   {"new"}, {"delete"}, {"?"}, {"::"}, {"."}, {".*"}, {"->"}, {"->*"}, {"~"},
+   {"{"}, {"}"}, {"["}, {"]"}, {"("}, {")"},
+   {"<:"}, {":>"}, {"<%"}, {"%>"}, {";"}, {":"}, {"..."},
+   {"?"}, {"::"}, {"."}, {".*"}, {"->"}, {"->*"}, {"~"},
    {"!"}, {"+"}, {"-"}, {"*"}, {"/"}, {"%"}, {"^"}, {"&"}, {"|"},
    {"="}, {"+="}, {"-="}, {"*="}, {"/="}, {"%="}, {"^="}, {"&="}, {"|="},
    {"=="}, {"!="}, {"<"}, {">"}, {"<="}, {">="}, {"<=>"}, {"&&"}, {"||"},
@@ -453,7 +541,7 @@ BNF GetCppBNFLex()
   } },
 
   { "r-char", {
-   all_except({")"}), // TODO: actually: non-match from "raw-string" above: ")", optional("d-char-sequence"), "\""
+   all_except({")"}), // TODO: actually: ")", optional("d-char-sequence"), "\""; with d-char-sequence like initial
   } },
 
   { "d-char-sequence", {
@@ -520,18 +608,8 @@ BNF GetCppBNFGram()
   // [gram.basic]
 
   {"translation-unit", {
-   {optional("top-level-declaration-seq")},
-   {optional("global-module-fragment"), "module-declaration", optional("top-level-declaration-seq"), optional("private-module-fragment")},
-  }},
- 
-  {"top-level-declaration-seq", {
-   {"top-level-declaration"},
-   {"top-level-declaration-seq", "top-level-declaration"},
-  }},
- 
-  {"top-level-declaration", {
-   {"module-import-declaration"},
-   {"declaration"},
+   {optional("declaration-seq")},
+   {optional("global-module-fragment"), "module-declaration", optional("declaration-seq"), optional("private-module-fragment")},
   }},
  
   // [gram.expr]
@@ -578,6 +656,9 @@ BNF GetCppBNFGram()
   }},
  
   {"lambda-introducer", {{"[", optional("lambda-capture"), "]"}}},
+
+  {"lambda-declarator", {{"(", "parameter-declaration-clause", ")", optional("decl-specifier-seq"),
+                          optional("noexcept-specifier"), optional("attribute-specifier-seq"), optional("trailing-return-type"), optional("requires-clause")}}},
  
   {"lambda-capture", {
    {"capture-default"},
@@ -593,31 +674,31 @@ BNF GetCppBNFGram()
   }},
  
   {"capture", {
-   {"simple-capture", optional("...")},
-   {optional("..."), "init-capture"},
+   {"simple-capture"},
+   {"init-capture"},
   }},
  
   {"simple-capture", {
-   {"identifier"},
-   {"&", "identifier"},
+   {"identifier", optional("...")},
+   {"&", "identifier", optional("...")},
    {"this"},
    {"*", "this"},
   }},
  
   {"init-capture", {
-   {"identifier", "initializer"},
-   {"&", "identifier", "initializer"},
+   {optional("..."), "identifier", "initializer"},
+   {"&", optional("..."), "identifier", "initializer"},
   }},
  
   {"fold-expression", {
-   {"(", "cast-expression", "fold-operator", ")"},
+   {"(", "cast-expression", "fold-operator", "...", ")"},
    {"(", "...", "fold-operator", "cast-expression", ")"},
    {"(", "cast-expression", "fold-operator", "...", "fold-operator", "cast-expression", ")"},
   }},
  
   {"fold-operator", {
    {"+"}, {"-"}, {"*"}, {"/"}, {"%"}, {"^"}, {"&"}, {"|"}, {"<<"}, {">>"},
-   {"+="}, {"-="}, {"*="}, {"/="}, {"%="}, {"^="}, {"&="}, {"|="}, {"<<="}, {">>="},
+   {"+="}, {"-="}, {"*="}, {"/="}, {"%="}, {"^="}, {"&="}, {"|="}, {"<<="}, {">>="}, {"="},
    {"=="}, {"!="}, {"<"}, {">"}, {"<="}, {">="}, {"&&"}, {"||"}, {","}, {".*"}, {"->*"},
   }},
  
@@ -927,6 +1008,7 @@ BNF GetCppBNFGram()
    {"namespace-definition"},
    {"empty-declaration"},
    {"attribute-declaration"},
+   {"module-import-declaration"},
   }},
 
   { "block-declaration", {
@@ -1186,7 +1268,7 @@ BNF GetCppBNFGram()
 
   { "parameter-declaration", {
    {optional("attribute-specifier-seq"), "decl-specifier-seq", "declarator"},
-   {optional("attribute-specifier-seq"), "decl-specifier-seq", "declarator", "initializer-clause"},
+   {optional("attribute-specifier-seq"), "decl-specifier-seq", "declarator", "=", "initializer-clause"},
    {optional("attribute-specifier-seq"), "decl-specifier-seq", optional("abstract-declarator")},
    {optional("attribute-specifier-seq"), "decl-specifier-seq", optional("abstract-declarator"), "=", "initializer-clause"},
   } },
@@ -1421,13 +1503,13 @@ BNF GetCppBNFGram()
    {"(", optional("balanced-token-seq"), ")"},
    {"[", optional("balanced-token-seq"), "]"},
    {"{", optional("balanced-token-seq"), "}"},
-   all_except({"(", ")", "[", "]", "{", "}",}),
+   all_except({"(", ")", "[", "]", "{", "}",}), // TODO: all tokens except ..., not characters!
   }},
 
   // [gram.module]
 
   { "module-declaration", {
-   {optional("export"), "module", "module-name", optional("module-partition"), optional("attribute-specifier-seq"), ";"},
+   {optional("export"), "module-keyword", "module-name", optional("module-partition"), optional("attribute-specifier-seq"), ";"},
   } },
 
   { "module-name", {
@@ -1446,20 +1528,21 @@ BNF GetCppBNFGram()
   { "export-declaration", {
    {"export", "declaration"},
    {"export", "{", optional("declaration-seq"), "}"},
+   {"export-keyword", "module-import-declaration"},
   } },
 
   { "module-import-declaration", {
-   {optional("export"), "import-keyword", "module-name",      optional("attribute-specifier-seq"), ";"},
-   {optional("export"), "import-keyword", "module-partition", optional("attribute-specifier-seq"), ";"},
-   {optional("export"), "import-keyword", "header-name",      optional("attribute-specifier-seq"), ";"},
+   {"import-keyword", "module-name",      optional("attribute-specifier-seq"), ";"},
+   {"import-keyword", "module-partition", optional("attribute-specifier-seq"), ";"},
+   {"import-keyword", "header-name",      optional("attribute-specifier-seq"), ";"},
   } },
 
   { "global-module-fragment", {
-   {"module", ";", optional("top-level-declaration-seq")},
+   {"module-keyword", ";", optional("declaration-seq")},
   } },
 
   { "private-module-fragment", {
-   {"module", ":", "private", ";", optional("top-level-declaration-seq")},
+   {"module-keyword", ":", "private", ";", optional("declaration-seq")},
   } },
 
   // [gram.class]
@@ -1492,7 +1575,6 @@ BNF GetCppBNFGram()
    {"union"},
   } },
 
-
   { "member-specification", {
    {"member-declaration", optional("member-specification")},
    {"access-specifier", ":", optional("member-specification")},
@@ -1553,7 +1635,7 @@ BNF GetCppBNFGram()
    {":", "base-specfier-list"},
   } },
 
-  { "base-specfier-list", {
+  { "base-specifier-list", {
    {"base-specifier", optional("...")},
    {"base-specifier-list", ",", "base-specifier", optional("...")},
   } },
@@ -1602,7 +1684,7 @@ BNF GetCppBNFGram()
   } },
 
   { "OPERATOR", {
-   {"new"}, {"delete"}, {"new[]"}, {"delete[]"}, {"co_await()[]"}, {"->"}, {"->*"},
+   {"new"}, {"delete"}, {"new[]"}, {"delete[]"}, {"co_await"}, {"()"}, {"[]"}, {"->"}, {"->*"},
    {"~"}, {"!"}, {"+"}, {"-"}, {"*"}, {"/"}, {"%"}, {"^"}, {"&"},
    {"|"}, {"="}, {"+="}, {"-="}, {"*="}, {"/="}, {"%="}, {"^="}, {"&="},
    {"|="}, {"=="}, {"!="}, {"<"}, {">"}, {"<="}, {">="}, {"<=>"}, {"&&"},
@@ -1654,8 +1736,8 @@ BNF GetCppBNFGram()
    {"type-parameter-key", optional("identifier"), "=", "type-id"},
    {"type-constraint", optional("...") ,optional("identifier")},
    {"type-constraint", optional("identifier"), "=", "type-id"},
-   {"template-head type-parameter-key", optional("..."), optional("identifier")},
-   {"template-head type-parameter-key", optional("identifier"), "=", "id-expression"},
+   {"template-head", "type-parameter-key", optional("..."), optional("identifier")},
+   {"template-head", "type-parameter-key", optional("identifier"), "=", "id-expression"},
   } },
 
   { "type-parameter-key", {
@@ -1755,8 +1837,21 @@ BNF GetCppBNFGram()
 
   { "preprocessing-file", {
    {optional("group")},
+   {"module-file"}
   } },
 
+  { "module-file", {
+   {optional("pp-global-module-fragment"), "pp-module", optional("group"), optional("pp-private-module-fragment")},
+  }},
+
+  { "pp-global-module-fragment", {
+   {"module", ";", "\n", optional("group")},
+  }},
+
+  { "pp-private-module-fragment", {
+   {"module", ":", "private", ";", "new-line", optional("group")},
+  }},
+
   { "group", {
    {"group-part"},
    {"group", "group-part"},
@@ -1771,7 +1866,7 @@ BNF GetCppBNFGram()
 
   { "control-line", {
    {"#", "include", "pp-tokens", "new-line"},
-   {optional("export"), "import", "pp-tokens", "new-line"},
+   {"pp-import"},
    {"#", "define", "identifier", "replacement-list", "new-line"},
    {"#", "define", "identifier", "lparen", optional("identifier-list"), ")", "replacement-list", "new-line"},
    {"#", "define", "identifier", "lparen", "...", ")", "replacement-list", "new-line"},
@@ -1824,7 +1919,7 @@ BNF GetCppBNFGram()
 
   { "identifier-list", {
    {"identifier"},
-   {"identifier-list", "identifier"},
+   {"identifier-list", ",", "identifier"},
   } },
 
   { "replacement-list", {
@@ -1868,34 +1963,16 @@ BNF GetCppBNFGram()
    {"__has_cpp_attribute", "(", "pp-tokens", ")"},
   } },
 
+  { "pp-module", {
+   {optional("export"), "module", optional("pp-tokens"), ";", "new-line"},
+  }},
+
   { "pp-import", {
    {optional("export"), "import", "header-name",        optional("pp-tokens"), ";", "new-line"},
    {optional("export"), "import", "header-name-tokens", optional("pp-tokens"), ";", "new-line"},
    {optional("export"), "import",                                "pp-tokens",  ";", "new-line"},
   } },
 
-  { "pp-global-module-fragment", {
-   {"module", ";", "pp-balanced-token-seq", "module"},
-  } },
-
-  { "pp-balanced-token-seq", {
-   {"pp-balanced-token"},
-   {"pp-balanced-token-seq", "pp-balanced-token"},
-  } },
-
-  { "pp-balanced-token", {
-   {"pp-ldelim", optional("pp-balanced-token-seq"), "pp-rdelim"},
-   // TODO: + any preprocessing-token except pp-ldelim and pp-rdelim
-  } },
-
-  { "pp-ldelim", {
-   {"(", "[", "{", "<:", "<%"},
-  } },
-
-  { "pp-rdelim", {
-   {")", "]", "}", ":>", "%>"},
-  } },
-
   { "va-opt-replacement", {
    {"__VA_OPT__", "(", optional("pp-tokens"), ")"},
   } },
diff --git a/tests/test-cpp.cpp b/tests/test-cpp.cpp
index e5b2a1a..adfa54b 100644
--- a/tests/test-cpp.cpp
+++ b/tests/test-cpp.cpp
@@ -24,7 +24,7 @@ class CppTest: public ::testing::Test
 {
 protected:
  CppTest() {
-  //debug = true;
+  debug = true;
  }
  ~CppTest() {
  }
@@ -42,7 +42,22 @@ TEST_F(CppTest, preprocessing_tokenize) {
 
  auto nodes = cpp.analysis(tokens);
  
- ASSERT_EQ(nodes.size(), 60/*44*/);
+ ASSERT_EQ(nodes.size(), 58/*44*/);
+}
+
+TEST_F(CppTest, preprocessing_tokenize_empty) {
+ CPP cpp;
+ auto pp_tokens = cpp.preprocessing_tokenize("");
+
+ ASSERT_EQ(pp_tokens.size(), 0);
+
+ auto tokens = cpp.tokens_from_pptokens(pp_tokens);
+
+ ASSERT_EQ(tokens.size(), 0);
+
+ auto nodes = cpp.analysis(tokens);
+ 
+ ASSERT_EQ(nodes.size(), 0);
 }
 
 TEST_F(CppTest, preprocessing_tokenize_compile_error) {
diff --git a/tests/test-grammer.cpp b/tests/test-grammer.cpp
index 1734da2..6af6992 100644
--- a/tests/test-grammer.cpp
+++ b/tests/test-grammer.cpp
@@ -50,7 +50,7 @@ TEST_F(GrammerTest, minimumSymbolsNeeded) {
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "logical-or-expression"), 1);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "assignment-expression"), 1);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "declaration"), 1);
- EXPECT_EQ(minimumSymbolsNeeded(compiler, "block-declaration"), 3);
+ EXPECT_EQ(minimumSymbolsNeeded(compiler, "block-declaration"), 2);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "simple-declaration"), 2);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "asm-declaration"), 5);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "namespace-alias-definition"), 5);
@@ -58,8 +58,8 @@ TEST_F(GrammerTest, minimumSymbolsNeeded) {
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "using-enum-declaration"), 4);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "using-directive"), 4);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "static_assert-declaration"), 5);
- EXPECT_EQ(minimumSymbolsNeeded(compiler, "alias-declaration"), 7);
+ EXPECT_EQ(minimumSymbolsNeeded(compiler, "alias-declaration"), 5);
  EXPECT_EQ(minimumSymbolsNeeded(compiler, "opaque-enum-declaration"), 3);
- EXPECT_EQ(minimumSymbolsNeeded(compiler, "function-definition"), 4);
+ EXPECT_EQ(minimumSymbolsNeeded(compiler, "function-definition"), 3);
 }