mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-11-09 20:07:43 +08:00
83 lines
2.7 KiB
Diff
83 lines
2.7 KiB
Diff
diff --git a/dev/lib/syntax.js b/dev/lib/syntax.js
|
||
index 904e27200ae4980183beda525d33feef70c2e7f7..f05dff0e41b478e856b8419a306dfba163e9a9af 100644
|
||
--- a/dev/lib/syntax.js
|
||
+++ b/dev/lib/syntax.js
|
||
@@ -662,7 +662,8 @@ function tokenizePath(effects, ok) {
|
||
if (
|
||
code === codes.eof ||
|
||
markdownLineEndingOrSpace(code) ||
|
||
- unicodeWhitespace(code)
|
||
+ unicodeWhitespace(code) ||
|
||
+ nonAsciiPunctuation(code)
|
||
) {
|
||
return ok(code)
|
||
}
|
||
@@ -762,7 +763,8 @@ function tokenizeTrail(effects, ok, nok) {
|
||
// So is whitespace.
|
||
code === codes.eof ||
|
||
markdownLineEndingOrSpace(code) ||
|
||
- unicodeWhitespace(code)
|
||
+ unicodeWhitespace(code) ||
|
||
+ nonAsciiPunctuation(code)
|
||
) {
|
||
return ok(code)
|
||
}
|
||
@@ -932,6 +934,17 @@ function previousEmail(code) {
|
||
return !(code === codes.slash || gfmAtext(code))
|
||
}
|
||
|
||
+/**
|
||
+ * Non-ASCII punctuation or symbol, such as `。` or `，`, which ends a literal
|
||
+ * in CJK prose.
|
||
+ *
|
||
+ * @param {Code} code
|
||
+ * @returns {boolean}
|
||
+ */
|
||
+function nonAsciiPunctuation(code) {
|
||
+ return code !== codes.eof && code > 127 && unicodePunctuation(code)
|
||
+}
|
||
+
|
||
/**
|
||
* @param {Code} code
|
||
* @returns {boolean}
|
||
diff --git a/lib/syntax.js b/lib/syntax.js
|
||
index dffbaf15146b34d3b3f634286629ccb148866f4c..00c336f17c389929d58dab156a72d8961a45caf8 100644
|
||
--- a/lib/syntax.js
|
||
+++ b/lib/syntax.js
|
||
@@ -584,7 +584,7 @@ function tokenizePath(effects, ok) {
|
||
if (code === 33 || code === 34 || code === 38 || code === 39 || code === 41 || code === 42 || code === 44 || code === 46 || code === 58 || code === 59 || code === 60 || code === 63 || code === 93 || code === 95 || code === 126) {
|
||
return effects.check(trail, ok, pathAtPunctuation)(code);
|
||
}
|
||
- if (code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code)) {
|
||
+ if (code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code) || nonAsciiPunctuation(code)) {
|
||
return ok(code);
|
||
}
|
||
effects.consume(code);
|
||
@@ -665,7 +665,7 @@ function tokenizeTrail(effects, ok, nok) {
|
||
// `<` is an end.
|
||
code === 60 ||
|
||
// So is whitespace.
|
||
- code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code)) {
|
||
+ code === null || markdownLineEndingOrSpace(code) || unicodeWhitespace(code) || nonAsciiPunctuation(code)) {
|
||
return ok(code);
|
||
}
|
||
return nok(code);
|
||
@@ -816,6 +816,17 @@ function previousEmail(code) {
|
||
return !(code === 47 || gfmAtext(code));
|
||
}
|
||
|
||
+/**
|
||
+ * Non-ASCII punctuation or symbol, such as `。` or `，`, which ends a literal
|
||
+ * in CJK prose.
|
||
+ *
|
||
+ * @param {Code} code
|
||
+ * @returns {boolean}
|
||
+ */
|
||
+function nonAsciiPunctuation(code) {
|
||
+ return code !== null && code > 127 && unicodePunctuation(code);
|
||
+}
|
||
+
|
||
/**
|
||
* @param {Code} code
|
||
* @returns {boolean}
|