YSTest: Lexical.cpp 源文件

YSTest PreAlpha_b900_20201012 The YSLib Test Project

Lexical.cpp
浏览该文件的文档.
1 /*
2 © 2012-2020 FrankHB.
3
4 This file is part of the YSLib project, and may only be used,
5 modified, and distributed under the terms of the YSLib project
6 license, LICENSE.TXT. By continuing to use, modify, or distribute
7 this file you indicate that you have read the license and
8 understand and accept it fully.
9 */
10
28 #include "NPL/YModules.h"
29 #include YFM_NPL_Lexical // for YSLib::octet;
30 #include <ystdex/string.hpp> // for ystdex::get_mid;
31
32 namespace NPL
33 {
34
35 bool
36 HandleBackslashPrefix(string_view buf, UnescapeContext& uctx)
37 {
38 YAssert(!buf.empty(), "Invalid buffer found.");
39 if(!uctx.IsHandling() && buf.back() == '\\')
40 {
41 yunseq(uctx.Start = buf.length() - 1, uctx.Length = 1);
42 return true;
43 }
44 return {};
45 }
46
47 bool
48 NPLUnescape(string& buf, char c, UnescapeContext& uctx, char ld)
49 {
50 if(uctx.Length == 1)
51 {
52 uctx.VerifyBufferLength(buf.length());
53
54 const auto i(uctx.Start);
55
56 YAssert(i == buf.length() - 1, "Invalid buffer found.");
57 switch(c)
58 {
59 case '\\':
60 buf[i] = '\\';
61 break;
62 case 'a':
63 buf[i] = '\a';
64 break;
65 case 'b':
66 buf[i] = '\b';
67 break;
68 case 'f':
69 buf[i] = '\f';
70 break;
71 case 'n':
72 buf[i] = '\n';
73 break;
74 case 'r':
75 buf[i] = '\r';
76 break;
77 case 't':
78 buf[i] = '\t';
79 break;
80 case 'v':
81 buf[i] = '\v';
82 break;
83 case '\n':
84 buf.pop_back();
85 break;
86 case '\'':
87 case '"':
88 if(c == ld)
89 {
90 buf[i] = ld;
91 break;
92 }
93 YB_ATTR_fallthrough;
94 default:
95 uctx.Clear();
96 buf += c;
97 return {};
98 }
99 uctx.Clear();
100 return true;
101 }
102 buf += c;
103 return {};
104 }
105
106
107 bool
108 LexicalAnalyzer::UpdateBack(char& b, char c)
109 {
110 switch(c)
111 {
112 case '\'':
113 case '"':
114 if(ld == char())
115 {
116 ld = c;
117 return true;
118 }
119 else if(ld == c)
120 {
121 ld = char();
122 return true;
123 }
124 break;
125 // XXX: Case ' ' is equivalent in the default branch.
126 // case ' ':
127 case '\f':
128 case '\n':
129 // XXX: Case '\r' is ignored.
130 // case '\r':
131 case '\t':
132 case '\v':
133 if(ld == char())
134 {
135 b = ' ';
136 break;
137 }
138 YB_ATTR_fallthrough;
139 default:
140 break;
141 }
142 return {};
143 }
144
145
146 char
147 CheckLiteral(string_view sv) ynothrowv
148 {
149 YAssertNonnull(sv.data());
150 if(sv.length() > 1)
151 if(const char c = ystdex::get_quote_mark_nonstrict(sv))
152 {
153 if(c == '\'' || c == '"')
154 return c;
155 }
156 return {};
157 }
158
159 string
160 Escape(string_view sv)
161 {
162 YAssertNonnull(sv.data());
163
164 char last{};
165 string res;
166
167 res.reserve(sv.length());
168 for(char c : sv)
169 {
170 char unescaped{};
171
172 switch(c)
173 {
174 case '\a':
175 unescaped = 'a';
176 break;
177 case '\b':
178 unescaped = 'b';
179 break;
180 case '\f':
181 unescaped = 'f';
182 break;
183 case '\n':
184 unescaped = 'n';
185 break;
186 case '\r':
187 unescaped = 'r';
188 break;
189 case '\t':
190 unescaped = 't';
191 break;
192 case '\v':
193 unescaped = 'v';
194 break;
195 case '\'':
196 case '"':
197 unescaped = c;
198 }
199 if(last == '\\')
200 {
201 if(c == '\\')
202 {
203 yunseq(last = char(), res += '\\');
204 continue;
205 }
206 switch(c)
207 {
208 case 'a':
209 case 'b':
210 case 'f':
211 case 'n':
212 case 'r':
213 case 't':
214 case 'v':
215 case '\'':
216 case '"':
217 res += '\\';
218 }
219 }
220 if(unescaped == char())
221 res += c;
222 else
223 {
224 res += '\\';
225 res += unescaped;
226 unescaped = char();
227 }
228 last = c;
229 }
230 return res;
231 }
232
233 string
234 EscapeLiteral(string_view sv)
235 {
236 const char c(CheckLiteral(sv));
237 // TODO: Use %get_mid for %string_view.
238 auto content(Escape(c == char() ? sv : ystdex::get_mid(string(sv))));
239
240 if(!content.empty() && content.back() == '\\')
241 content += '\\';
242 return c == char() ? std::move(content) : ystdex::quote(content, c);
243 }
244
245 string
246 EscapeXML(string_view sv)
247 {
248 YAssertNonnull(sv.data());
249
250 string res;
251
252 res.reserve(sv.length());
253 for(char c : sv)
254 switch(c)
255 {
256 case '\0':
257 YTraceDe(YSLib::Warning, "Invalid character '#x%X' found, ignored.",
258 unsigned(c));
259 break;
260 case '&':
261 res += "&amp;";
262 break;
263 case '<':
264 res += "&lt;";
265 break;
266 case '>':
267 res += "&gt;";
268 break;
269 default:
270 res += c;
271 }
272 return res;
273 }
274
275 // XXX: The code generated by x86_64-pc-linux G++ 8.3 at '-O3' is wrong. This
276 // seems not filed at https://gcc.gnu.org/bugzilla.
277 // XXX: There is no need to specifiy GCC version because the lowest version
278 // supported in YFramework is more recent. See
279 // https://gcc.gnu.org/onlinedocs/gcc-4.4.0/gcc/Function-Attributes.html#Function-Attributes.
280 #if (YB_IMPL_GNUCPP < 90100 && !YB_IMPL_CLANGPP) && YCL_Linux && \
281 defined(NDEBUG) && __OPTIMIZE__ && !__OPTIMIZE_SIZE__
282 // XXX: As there is no way to easily get the optimization level, leave all
283 // optimized builds as '-O2'. All '-O2' options plus all documented additional
284 // options for '-O3' or options used in 'gcc/opts.c' in trunk code as of
285 // writing (2019-05-16, also cf.
286 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81191) does not reappear the
287 // bug, but as now just use '-O2' instead.
288 YB_ATTR(optimize("O2"))
289 #endif
290 string
291 Literalize(string_view sv, char c)
292 {
293 return CheckLiteral(sv) == char() && c != char()
294 ? ystdex::quote(string(sv), c) : string(sv);
295 }
296
297
298 LexemeList
299 Decompose(string_view src, LexemeList::allocator_type a)
300 {
301 YAssertNonnull(src.data());
302
303 LexemeList dst(a);
304 using iter_type = typename string_view::const_iterator;
305
306 ystdex::split_l(src.cbegin(), src.cend(), IsDelimiter,
307 // TODO: Blocked. Use C++14 generic lambda expressions.
308 [&](iter_type b, iter_type e){
309 YAssert(e >= b, "Invalid split result found.");
310
311 string_view sv(&*b, size_t(e - b));
312
313 YAssert(!sv.empty(), "Null token found.");
314 if(IsGraphicalDelimiter(*b))
315 {
316 dst.push_back({sv.front()});
317 sv.remove_prefix(1);
318 }
319 ystdex::trim(sv);
320 if(!sv.empty())
321 dst.push_back({sv.data(), sv.size()});
322 });
323 return dst;
324 }
325
326
328 namespace
329 {
330
331 template<typename _fAdd, typename _fAppend, class _tParseResult>
332 void
333 UpdateByteRaw(_fAdd add, _fAppend append, _tParseResult& res, bool got_delim,
334 const LexicalAnalyzer& lexer, string& cbuf, bool& update_current)
335 {
336 const auto len(cbuf.length());
337
338 YAssert(!(res.empty() && update_current), "Invalid state found.");
339 if(len > 0 && !lexer.GetUnescapeContext().IsHandling())
340 {
341 if(len == 1)
342 {
343 const auto update_c([&](char c){
344 if(update_current)
345 append(res, c);
346 else
347 add(res, string({c}, res.get_allocator()));
348 });
349 const char c(cbuf.back());
350 const bool unquoted(lexer.GetDelimiter() == char());
351
352 if(got_delim)
353 {
354 update_c(c);
355 update_current = !unquoted;
356 }
357 else if(unquoted && IsDelimiter(c))
358 {
359 if(IsGraphicalDelimiter(c))
360 add(res, string({c}, res.get_allocator()));
361 update_current = {};
362 }
363 else
364 {
365 update_c(c);
366 update_current = true;
367 }
368 }
369 else if(update_current)
370 append(res, std::move(cbuf));
371 else
372 add(res, std::move(cbuf));
373 cbuf.clear();
374 }
375 }
376
377 template<class _tParseResult>
378 struct SequenceAdd final
379 {
380 template<typename _tParam>
381 void
382 operator()(_tParseResult& res, _tParam&& arg) const
383 {
384 res.push_back(yforward(arg));
385 }
386 };
387
388 template<class _tParseResult>
389 struct SequenceAppend final
390 {
391 template<typename _tParam>
392 void
393 operator()(_tParseResult& res, _tParam&& arg) const
394 {
395 res.back() += yforward(arg);
396 }
397 };
398
399 template<class _tParseResult>
400 struct SourcedSequenceAdd final
401 {
402 const SourcedByteParser& Parser;
403
404 template<typename _tParam>
405 void
406 operator()(_tParseResult& res, _tParam&& arg) const
407 {
408 res.emplace_back(Parser.GetSourceLocation(), yforward(arg));
409 }
410 };
411
412 template<class _tParseResult>
413 struct SourcedSequenceAppend final
414 {
415 template<typename _tParam>
416 void
417 operator()(_tParseResult& res, _tParam&& arg) const
418 {
419 res.back().second += yforward(arg);
420 }
421 };
422
423 } // unnamed namespace;
424
425
426 void
427 ByteParser::Update(bool got_delim)
428 {
429 // TODO: Blocked. Use C++14 generic lambda expressions.
430 UpdateByteRaw(SequenceAdd<ParseResult>(), SequenceAppend<ParseResult>(),
431 lexemes, got_delim, GetLexerRef(), GetBufferRef(), update_current);
432 }
433
434
435 void
436 SourcedByteParser::Update(bool got_delim)
437 {
438 // TODO: Blocked. Use C++14 generic lambda expressions.
439 UpdateByteRaw(SourcedSequenceAdd<ParseResult>{*this},
440 SourcedSequenceAppend<ParseResult>(), lexemes, got_delim, GetLexerRef(),
441 GetBufferRef(), update_current);
442 }
443
444
445 LexemeList
446 DelimitedByteParser::GetResult() const
447 {
448 const auto& cbuf(GetBuffer());
449 const auto a(cbuf.get_allocator());
450 LexemeList res(a);
451 size_t i(0);
452
453 std::for_each(qlist.cbegin(), qlist.cend(), [&](size_t s){
454 if(s != i)
455 {
456 DecomposeString(res, string_view(&cbuf[i], s - i));
457 i = s;
458 }
459 });
460 if(cbuf.length() != i)
461 DecomposeString(res, string_view(&cbuf[i], cbuf.length() - i));
462 return res;
463 }
464
465 void
466 DelimitedByteParser::DecomposeString(LexemeList& lst, bool not_quoted,
467 string_view sv)
468 {
469 if(not_quoted)
470 lst.splice(lst.end(), Decompose(sv, lst.get_allocator()));
471 else
472 lst.push_back(LexemeList::value_type(sv.begin(), sv.end()));
473 }
474
475 } // namespace NPL;
476
NPL::EscapeLiteral
YB_ATTR_nodiscard YF_API YB_PURE string EscapeLiteral(string_view)
编码转义字符串字面量。
Definition: Lexical.cpp:234
CHRLib::e
const char const char * e
Definition: CharacterProcessing.h:112
string.hpp
ISO C++ 标准字符串扩展。
NPL::YAssert
YB_ATTR_nodiscard YB_PURE yconstfn YAssert(!buf.empty(), "Invalid buffer found.")
CHRLib::c
c
Definition: CharacterProcessing.h:84
ystdex::quote
decay_t< _tString > quote(_tString &&str, typename string_traits< decay_t< _tString >>::value_type c=typename string_traits< decay_t< _tString >>::value_type('"'))
取添加前缀和后缀的字符串。
Definition: string.hpp:1579
NPL::Decompose
LexemeList Decompose(string_view src, LexemeList::allocator_type a)
Definition: Lexical.cpp:299
NPL::LexicalAnalyzer::ld
char ld
字面分隔符状态：表示正在处理字面量中的有效字符。
Definition: Lexical.h:222
ystdex::get_mid
_tString get_mid(const _tString &str, typename _tString::size_type l=1)
取删除前缀和后缀的子字符串。
Definition: string.hpp:1608
NPL::LexicalAnalyzer::char
char
Definition: Lexical.h:231
ystdex::list::splice
void splice(const_iterator position, list &x)
Definition: list.hpp:1502
ystdex::cpp2017::string_view
basic_string_view< char > string_view
Definition: string_view.hpp:437
NPL::HandleBackslashPrefix
YB_ATTR_nodiscard YF_API bool HandleBackslashPrefix(string_view, UnescapeContext &)
设置反斜杠转义前缀：当输入 '\' 时设置前缀为 "\\" 。
Definition: Lexical.cpp:36
NPL::string_view
YB_ATTR_nodiscard ctx string_view
Definition: NPLA.h:3020
YB_ATTR_fallthrough
#define YB_ATTR_fallthrough
用于 switch 语句的 fallthrough 属性。
Definition: ydef.h:655
NPL::CheckLiteral
YB_ATTR_nodiscard YF_API YB_PURE char CheckLiteral(string_view) ynothrowv
检查指定字符串是否为字面量。
Definition: Lexical.cpp:147
YAssert
#define YAssert(_expr, _msg)
Definition: cassert.h:154
NPL::Escape
string Escape(string_view sv)
Definition: Lexical.cpp:160
NPL::NPLUnescape
YB_ATTR_nodiscard YB_PURE yconstfn false YB_ATTR_nodiscard YF_API bool NPLUnescape(string &, char, UnescapeContext &, char)
NPL 转义匹配算法。
Definition: Lexical.cpp:48
YModules.h
NPL 库模块配置文件。
ystdex::list::allocator_type
_tAlloc allocator_type
Definition: list.hpp:1064
yforward
#define yforward(_expr)
根据参数类型使用 std::forward 转发对应参数。
Definition: ydef.h:1508
YSLib::Drawing::src
src
Definition: YGDI.h:442
YB_ATTR
#define YB_ATTR(...)
GNU 风格属性。
Definition: ydef.h:567
YAssertNonnull
#define YAssertNonnull(_expr)
Definition: cassert.h:165
ystdex::list::value_type
_type value_type
Definition: list.hpp:1053
YSLib::i
return i
Definition: YCoreUtilities.h:177
ystdex::trim
_tString && trim(_tString &&str, typename string_traits< _tString >::value_type c)
删除字符串中指定的连续前缀与后缀字符。
Definition: string.hpp:1485
NPL::UnescapeContext
反转义上下文。
Definition: Lexical.h:119
yunseq
#define yunseq
无序列依赖表达式组求值。
Definition: ydef.h:1535
YTraceDe
#define YTraceDe(...)
YCLib 默认调试跟踪。
Definition: Debug.h:407
NPL::EscapeXML
YB_ATTR_nodiscard YF_API YB_PURE string EscapeXML(string_view)
编码 XML 字符串。
Definition: Lexical.cpp:246
NPL
Definition: NPL/Configuration.h:35
platform::Descriptions::Warning
@ Warning
Definition: YCommon.h:183
CHRLib::s
_tDst const _tSrc * s
Definition: CharacterProcessing.h:143
ynothrowv
#define ynothrowv
YSLib 无异常抛出保证验证：有条件地使用无异常抛出规范。
Definition: ydef.h:1219
YSLib::Text::buf
YB_ATTR_nodiscard YB_PURE const char * buf
Definition: TextFile.h:83
NPL::UnescapeContext::Length
size_t Length
转义序列长度。
Definition: Lexical.h:133
ystdex::list::end
iterator end()
Definition: list.hpp:1242
NPL::Literalize
YB_ATTR_nodiscard YF_API YB_PURE string Literalize(string_view, char='"')
修饰字符串为字面量。
Definition: Lexical.cpp:291
ystdex::get_quote_mark_nonstrict
string_traits< _tString >::value_type get_quote_mark_nonstrict(const _tString &str)
取相同的前缀和后缀元素。
Definition: string.hpp:1564
NPL::UnescapeContext::Start
size_t Start
转义序列起始位置。
Definition: Lexical.h:131
NPL::dst
YF_API const char src const string const char * dst
Definition: Dependency.h:96
ystdex::cpp2017::string
basic_string< char > string
Definition: string.hpp:772
ystdex::list::push_back
void push_back(const value_type &x)
Definition: list.hpp:1402
ystdex::list::get_allocator
allocator_type get_allocator() const
Definition: list.hpp:1225
NPL::LexemeList
list< string > LexemeList
词素列表：待被处理为记号短字符串列表。
Definition: Lexical.h:75
ystdex::list
列表容器。
Definition: list.hpp:1051
YSLib::a
YB_PURE const _type * a
Definition: YCoreUtilities.h:168
NPL::_tParam
_tParam
Definition: NPLA.h:2367
ystdex::split_l
_tIn split_l(_tIn b, _tIn e, _fPred is_delim, _fInsert insert)
以指定分隔符分割保留起始分隔符的序列。
Definition: algorithm.hpp:339