From 67960d229844095a023698b7387e424737bf3cf8 Mon Sep 17 00:00:00 2001 From: samstrohkorbatt Date: Mon, 26 Feb 2024 09:04:22 -0800 Subject: [PATCH 1/3] Adding python f-string syntax support --- src/basic-languages/python/python.test.ts | 66 ++++++++++++++++++++++- src/basic-languages/python/python.ts | 24 +++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/basic-languages/python/python.test.ts b/src/basic-languages/python/python.test.ts index 579c4415..cbd64d69 100644 --- a/src/basic-languages/python/python.test.ts +++ b/src/basic-languages/python/python.test.ts @@ -189,5 +189,69 @@ testTokenization('python', [ line: '456.7e-7j', tokens: [{ startIndex: 0, type: 'number.python' }] } - ] + ], + + // F-Strings + [ + { + line: 'f"str {var} str"', + tokens: [ + {startIndex: 0, type: 'string.escape.python'}, + {startIndex: 2, type: 'string.python'}, + {startIndex: 6, type: 'identifier.python'}, + {startIndex: 11, type: 'string.python'}, + {startIndex: 15, type: 'string.escape.python'} + ] + } + ], + [ + { + line: `f'''str {var} str'''`, + tokens: [ + {startIndex: 0, type: 'string.escape.python'}, + {startIndex: 4, type: 'string.python'}, + {startIndex: 8, type: 'identifier.python'}, + {startIndex: 13, type: 'string.python'}, + {startIndex: 17, type: 'string.escape.python'} + ] + } + ], + [ + { + line: 'f"{var:.3f}{var!r}{var=}"', + tokens: [ + {startIndex: 0, type: 'string.escape.python'}, + {startIndex: 2, type: 'identifier.python'}, + {startIndex: 6, type: 'string.python'}, + {startIndex: 10, type: 'identifier.python'}, + {startIndex: 15, type: 'string.python'}, + {startIndex: 17, type: 'identifier.python'}, + {startIndex: 22, type: 'string.python'}, + {startIndex: 23, type: 'identifier.python'}, + {startIndex: 24, type: 'string.escape.python'} + ] + } + ], + [ + { + line: 'f"\' " "', + tokens: [ + { startIndex: 0, type: 'string.escape.python' }, + { startIndex: 2, type: 'string.python' }, + { startIndex: 4, type: 'string.escape.python' }, + { startIndex: 5, type: 'white.python' }, + { startIndex: 6, type: 'string.escape.python' } + ] + } + ], + [ + { + line: '"{var}"', + tokens: [ + { startIndex: 0, type: 'string.escape.python' }, + { startIndex: 1, type: 'string.python' }, + { startIndex: 6, type: 'string.escape.python' } + ] + } + ], ]); diff --git a/src/basic-languages/python/python.ts b/src/basic-languages/python/python.ts index c9d43a46..a4d0b51b 100644 --- a/src/basic-languages/python/python.ts +++ b/src/basic-languages/python/python.ts @@ -250,10 +250,20 @@ export const language = { // Recognize strings, including those broken across lines with \ (but not without) strings: [ [/'$/, 'string.escape', '@popall'], + [/f'{1,3}/, 'string.escape', '@fStringBody'], [/'/, 'string.escape', '@stringBody'], [/"$/, 'string.escape', '@popall'], + [/f"{1,3}/, 'string.escape', '@fDblStringBody'], [/"/, 'string.escape', '@dblStringBody'] ], + fStringBody: [ + [/[^\\'\{\}]+$/, 'string', '@popall'], + [/[^\\'\{\}]+/, 'string'], + [/\{[^\}':!=]+/, 'identifier', '@fStringDetail'], + [/\\./, 'string'], + [/'/, 'string.escape', '@popall'], + [/\\$/, 'string'] + ], stringBody: [ [/[^\\']+$/, 'string', '@popall'], [/[^\\']+/, 'string'], @@ -261,12 +271,26 @@ export const language = { [/'/, 'string.escape', '@popall'], [/\\$/, 'string'] ], + fDblStringBody: [ + [/[^\\"\{\}]+$/, 'string', '@popall'], + [/[^\\"\{\}]+/, 'string'], + [/\{[^\}':!=]+/, 'identifier', '@fStringDetail'], + [/\\./, 'string'], + [/"/, 'string.escape', '@popall'], + [/\\$/, 'string'] + ], dblStringBody: [ [/[^\\"]+$/, 'string', '@popall'], [/[^\\"]+/, 'string'], [/\\./, 'string'], [/"/, 'string.escape', '@popall'], [/\\$/, 'string'] + ], + fStringDetail: [ + [/[:][^}]+/, 'string'], + [/[!][ars]/, 'string'], // only !a, !r, !s are supported by f-strings: https://docs.python.org/3/tutorial/inputoutput.html#formatted-string-literals + [/=/, 'string'], + [/\}/, 'identifier', '@pop'] ] } }; From b3ad68b237ca09dfe6d79c747c08886cbdc186f7 Mon Sep 17 00:00:00 2001 From: samstrohkorbatt Date: Fri, 1 Mar 2024 09:29:02 -0700 Subject: [PATCH 2/3] retrigger checks From 77e16ebc8139f4f6cea3332b97e155fe8f9fde08 Mon Sep 17 00:00:00 2001 From: samstrohkorbatt Date: Mon, 1 Apr 2024 14:42:16 -0600 Subject: [PATCH 3/3] Fixing formatting --- src/basic-languages/python/python.test.ts | 40 +++++++++++------------ src/basic-languages/python/python.ts | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/basic-languages/python/python.test.ts b/src/basic-languages/python/python.test.ts index cbd64d69..019758a4 100644 --- a/src/basic-languages/python/python.test.ts +++ b/src/basic-languages/python/python.test.ts @@ -196,11 +196,11 @@ testTokenization('python', [ { line: 'f"str {var} str"', tokens: [ - {startIndex: 0, type: 'string.escape.python'}, - {startIndex: 2, type: 'string.python'}, - {startIndex: 6, type: 'identifier.python'}, - {startIndex: 11, type: 'string.python'}, - {startIndex: 15, type: 'string.escape.python'} + { startIndex: 0, type: 'string.escape.python' }, + { startIndex: 2, type: 'string.python' }, + { startIndex: 6, type: 'identifier.python' }, + { startIndex: 11, type: 'string.python' }, + { startIndex: 15, type: 'string.escape.python' } ] } ], @@ -208,11 +208,11 @@ testTokenization('python', [ { line: `f'''str {var} str'''`, tokens: [ - {startIndex: 0, type: 'string.escape.python'}, - {startIndex: 4, type: 'string.python'}, - {startIndex: 8, type: 'identifier.python'}, - {startIndex: 13, type: 'string.python'}, - {startIndex: 17, type: 'string.escape.python'} + { startIndex: 0, type: 'string.escape.python' }, + { startIndex: 4, type: 'string.python' }, + { startIndex: 8, type: 'identifier.python' }, + { startIndex: 13, type: 'string.python' }, + { startIndex: 17, type: 'string.escape.python' } ] } ], @@ -220,15 +220,15 @@ testTokenization('python', [ { line: 'f"{var:.3f}{var!r}{var=}"', tokens: [ - {startIndex: 0, type: 'string.escape.python'}, - {startIndex: 2, type: 'identifier.python'}, - {startIndex: 6, type: 'string.python'}, - {startIndex: 10, type: 'identifier.python'}, - {startIndex: 15, type: 'string.python'}, - {startIndex: 17, type: 'identifier.python'}, - {startIndex: 22, type: 'string.python'}, - {startIndex: 23, type: 'identifier.python'}, - {startIndex: 24, type: 'string.escape.python'} + { startIndex: 0, type: 'string.escape.python' }, + { startIndex: 2, type: 'identifier.python' }, + { startIndex: 6, type: 'string.python' }, + { startIndex: 10, type: 'identifier.python' }, + { startIndex: 15, type: 'string.python' }, + { startIndex: 17, type: 'identifier.python' }, + { startIndex: 22, type: 'string.python' }, + { startIndex: 23, type: 'identifier.python' }, + { startIndex: 24, type: 'string.escape.python' } ] } ], @@ -253,5 +253,5 @@ testTokenization('python', [ { startIndex: 6, type: 'string.escape.python' } ] } - ], + ] ]); diff --git a/src/basic-languages/python/python.ts b/src/basic-languages/python/python.ts index a4d0b51b..a209212c 100644 --- a/src/basic-languages/python/python.ts +++ b/src/basic-languages/python/python.ts @@ -288,7 +288,7 @@ export const language = { ], fStringDetail: [ [/[:][^}]+/, 'string'], - [/[!][ars]/, 'string'], // only !a, !r, !s are supported by f-strings: https://docs.python.org/3/tutorial/inputoutput.html#formatted-string-literals + [/[!][ars]/, 'string'], // only !a, !r, !s are supported by f-strings: https://docs.python.org/3/tutorial/inputoutput.html#formatted-string-literals [/=/, 'string'], [/\}/, 'identifier', '@pop'] ]