Implementing section access expression

src/powerquery-parser/language/ast/ast.ts

@@ -58,6 +58,7 @@ export enum NodeKind {
     RecursivePrimaryExpression = "RecursivePrimaryExpression",
     RelationalExpression = "RelationalExpression",
     Section = "Section",
+    SectionAccessExpression = "SectionAccessExpression",
     SectionMember = "SectionMember",
     TableType = "TableType",
     TypePrimaryType = "TypePrimaryType",
@@ -346,7 +347,7 @@ export type TPrimaryExpression =
     | ListExpression
     | RecordExpression
     | IdentifierExpression
-    // SectionAccessExpression
+    | SectionAccessExpression
     | ParenthesizedExpression
     | TFieldAccessExpression
     | TRecursivePrimaryExpression
@@ -383,6 +384,18 @@ export interface IdentifierExpression extends INode {
     readonly identifier: Identifier;
 }
 
+// ---------------------------------------------------------
+// ---------- 12.2.3.13 Section-access expression ----------
+// ---------------------------------------------------------
+
+export interface SectionAccessExpression extends INode {
+    readonly kind: NodeKind.SectionAccessExpression;
+    readonly isLeaf: false;
+    readonly sectionIdentifier: Identifier;
+    readonly bangConstant: IConstant<Constant.MiscConstant.Bang>;
+    readonly memberIdentifier: Identifier;
+}
+
 // --------------------------------------------------------
 // ---------- 12.2.3.14 Parenthesized expression ----------
 // --------------------------------------------------------
@@ -919,6 +932,7 @@ export const NodeKindsForTPrimaryExpression: Set<NodeKind> = new Set([
     NodeKind.ListExpression,
     NodeKind.RecordExpression,
     NodeKind.IdentifierExpression,
+    NodeKind.SectionAccessExpression,
     NodeKind.ParenthesizedExpression,
     NodeKind.InvokeExpression,
     NodeKind.RecursivePrimaryExpression,

src/powerquery-parser/language/ast/astUtils/astUtils.ts

@@ -265,6 +265,7 @@ export function simplifyType(type: Ast.TType): SimplifiedType {
         case Ast.NodeKind.RecordExpression:
         case Ast.NodeKind.RecursivePrimaryExpression:
         case Ast.NodeKind.RelationalExpression:
+        case Ast.NodeKind.SectionAccessExpression:
         case Ast.NodeKind.TypePrimaryType:
         case Ast.NodeKind.UnaryExpression:
             throw new CommonError.InvariantError("this should never be reached", {

src/powerquery-parser/language/ast/astUtils/typeGuards.ts

@@ -90,6 +90,7 @@ export function isTBinOpExpressionKind(nodeKind: Ast.NodeKind): nodeKind is Ast.
         case Ast.NodeKind.RecordType:
         case Ast.NodeKind.RecursivePrimaryExpression:
         case Ast.NodeKind.Section:
+        case Ast.NodeKind.SectionAccessExpression:
         case Ast.NodeKind.SectionMember:
         case Ast.NodeKind.TableType:
         case Ast.NodeKind.TypePrimaryType:
@@ -160,6 +161,7 @@ export function isTKeyValuePair(node: Ast.TNode): node is Ast.TKeyValuePair {
         case Ast.NodeKind.RecursivePrimaryExpression:
         case Ast.NodeKind.RelationalExpression:
         case Ast.NodeKind.Section:
+        case Ast.NodeKind.SectionAccessExpression:
         case Ast.NodeKind.SectionMember:
         case Ast.NodeKind.TableType:
         case Ast.NodeKind.TypePrimaryType:
@@ -231,6 +233,7 @@ export function isTPairedConstant(node: Ast.TNode): node is Ast.TPairedConstant
         case Ast.NodeKind.RecursivePrimaryExpression:
         case Ast.NodeKind.RelationalExpression:
         case Ast.NodeKind.Section:
+        case Ast.NodeKind.SectionAccessExpression:
         case Ast.NodeKind.SectionMember:
         case Ast.NodeKind.TableType:
         case Ast.NodeKind.UnaryExpression:

src/powerquery-parser/language/constant/constant.ts

@@ -57,6 +57,7 @@ export enum KeywordConstant {
 export enum MiscConstant {
     Ampersand = "&",
     AtSign = "@",
+    Bang = "!",
     Comma = ",",
     DotDot = "..",
     Ellipsis = "...",
@@ -181,6 +182,7 @@ export const LogicalOperators: ReadonlyArray<LogicalOperator> = [LogicalOperator
 export const MiscConstants: ReadonlyArray<MiscConstant> = [
     MiscConstant.Ampersand,
     MiscConstant.AtSign,
+    MiscConstant.Bang,
     MiscConstant.Comma,
     MiscConstant.DotDot,
     MiscConstant.Ellipsis,

src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/specializedSelectors.ts

@@ -1,11 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { assertNthChildXor, assertNthChildXorChecked, nthChildAstChecked, nthChildXorChecked } from "./childSelectors";
+import { assertNthChildXor, assertNthChildXorChecked, nthChildAstChecked, nthChildXor } from "./childSelectors";
 import { assertParentXor, assertParentXorChecked } from "./parentSelectors";
 import { assertXor, assertXorChecked } from "./commonSelectors";
 import { NodeIdMap, NodeIdMapUtils, XorNodeUtils } from "..";
-import { TXorNode, XorNode } from "../xorNode";
+import { AstXorNode, TXorNode, XorNode } from "../xorNode";
 import { Ast } from "../../../language";
 import { CommonError } from "../../../common";
 
@@ -49,11 +49,11 @@ export function assertRecursiveExpressionPreviousSibling<T extends Ast.TNode>(
 
         return expectedNodeKinds
             ? assertNthChildXorChecked(
-                  nodeIdMapCollection,
-                  arrayWrapper.node.id,
-                  indexOfPrimaryExpressionId - 1,
-                  expectedNodeKinds,
-              )
+                nodeIdMapCollection,
+                arrayWrapper.node.id,
+                indexOfPrimaryExpressionId - 1,
+                expectedNodeKinds,
+            )
             : assertNthChildXor(nodeIdMapCollection, arrayWrapper.node.id, indexOfPrimaryExpressionId - 1);
     }
     // It's the first element in ArrayWrapper, meaning we must grab RecursivePrimaryExpression.head
@@ -73,32 +73,42 @@ export function invokeExpressionIdentifier(
     nodeId: number,
 ): XorNode<Ast.IdentifierExpression> | undefined {
     const invokeExprXorNode: TXorNode = assertXorChecked(nodeIdMapCollection, nodeId, Ast.NodeKind.InvokeExpression);
+    const headXorNode: TXorNode | undefined = invokeExpressionHead(nodeIdMapCollection, invokeExprXorNode);
 
-    // The only place for an identifier in a RecursivePrimaryExpression is as the head, therefore an InvokeExpression
-    // only has a name if the InvokeExpression is the 0th element in the RecursivePrimaryExpressionArray.
+    return headXorNode ? identifierExpressionHead(invokeExprXorNode, headXorNode) : undefined;
+}
+
+// The only place for a name in a RecursivePrimaryExpression is as the head, therefore an InvokeExpression
+// only has a name if the InvokeExpression is the 0th element in the RecursivePrimaryExpressionArray.
+function invokeExpressionHead(
+    nodeIdMapCollection: NodeIdMap.Collection,
+    invokeExprXorNode: TXorNode,
+): TXorNode | undefined {
     if (invokeExprXorNode.node.attributeIndex !== 0) {
         return undefined;
     }
 
-    // Grab the RecursivePrimaryExpression's head if it's an IdentifierExpression
     const recursiveArrayXorNode: TXorNode = assertParentXor(nodeIdMapCollection, invokeExprXorNode.node.id);
     const recursiveExprXorNode: TXorNode = assertParentXor(nodeIdMapCollection, recursiveArrayXorNode.node.id);
 
-    const headXorNode: XorNode<Ast.IdentifierExpression> | undefined = nthChildXorChecked<Ast.IdentifierExpression>(
+    return nthChildXor(
         nodeIdMapCollection,
         recursiveExprXorNode.node.id,
-        0,
-        Ast.NodeKind.IdentifierExpression,
+        0
     );
+}
 
+// Throws if the IdentifierExpression hasn't finished parsing.
+function identifierExpressionHead(
+    invokeExprXorNode: TXorNode,
+    headXorNode: TXorNode,
+): AstXorNode<Ast.IdentifierExpression> | undefined {
     // It's not an identifier expression so there's nothing we can do.
-    if (headXorNode === undefined) {
+    if (!XorNodeUtils.isNodeKind<Ast.IdentifierExpression>(headXorNode, Ast.NodeKind.IdentifierExpression)) {
         return undefined;
     }
 
-    // The only place for an identifier in a RecursivePrimaryExpression is as the head, therefore an InvokeExpression
-    // only has a name if the InvokeExpression is the 0th element in the RecursivePrimaryExpressionArray.
-    if (XorNodeUtils.isContext(headXorNode)) {
+    if (!XorNodeUtils.isAstChecked<Ast.IdentifierExpression>(headXorNode, Ast.NodeKind.IdentifierExpression)) {
         const details: {
             identifierExpressionNodeId: number;
             invokeExpressionNodeId: number;
@@ -117,26 +127,43 @@ export function invokeExpressionIdentifier(
 }
 
 // Unboxes the identifier literal for function name if it exists.
+// A section-access head (eg. `Section1!Foo(1)`) is returned as `Section1!Foo`.
 export function invokeExpressionIdentifierLiteral(
     nodeIdMapCollection: NodeIdMap.Collection,
     nodeId: number,
 ): string | undefined {
-    assertXorChecked(nodeIdMapCollection, nodeId, Ast.NodeKind.InvokeExpression);
+    const invokeExprXorNode: TXorNode = assertXorChecked(nodeIdMapCollection, nodeId, Ast.NodeKind.InvokeExpression);
+    const headXorNode: TXorNode | undefined = invokeExpressionHead(nodeIdMapCollection, invokeExprXorNode);
 
-    const identifierExpressionXorNode: XorNode<Ast.IdentifierExpression> | undefined = invokeExpressionIdentifier(
-        nodeIdMapCollection,
-        nodeId,
+    if (headXorNode === undefined) {
+        return undefined;
+    }
+
+    const identifierExpressionXorNode: AstXorNode<Ast.IdentifierExpression> | undefined = identifierExpressionHead(
+        invokeExprXorNode,
+        headXorNode,
     );
 
-    if (identifierExpressionXorNode === undefined || XorNodeUtils.isContext(identifierExpressionXorNode)) {
+    if (identifierExpressionXorNode !== undefined) {
+        const identifierExpression: Ast.IdentifierExpression = identifierExpressionXorNode.node;
+
+        return identifierExpression.inclusiveConstant === undefined
+            ? identifierExpression.identifier.literal
+            : identifierExpression.inclusiveConstant.constantKind + identifierExpression.identifier.literal;
+    }
+
+    // A SectionAccessExpression head which is still being parsed is treated as having no name.
+    if (!XorNodeUtils.isAstChecked<Ast.SectionAccessExpression>(headXorNode, Ast.NodeKind.SectionAccessExpression)) {
         return undefined;
     }
 
-    const identifierExpression: Ast.IdentifierExpression = identifierExpressionXorNode.node;
+    const sectionAccessExpression: Ast.SectionAccessExpression = headXorNode.node;
 
-    return identifierExpression.inclusiveConstant === undefined
-        ? identifierExpression.identifier.literal
-        : identifierExpression.inclusiveConstant.constantKind + identifierExpression.identifier.literal;
+    return (
+        sectionAccessExpression.sectionIdentifier.literal +
+        sectionAccessExpression.bangConstant.constantKind +
+        sectionAccessExpression.memberIdentifier.literal
+    );
 }
 
 // Unboxes the node if it's a identifier

src/powerquery-parser/parser/parseState/parseStateUtils.ts

@@ -262,14 +262,23 @@ export function isOnGeneralizedIdentifierStart(state: ParseState, tokenIndex: nu
     }
 }
 
+// section-access-expression: identifier ! identifier
+export function isSectionAccessExpressionNext(state: ParseState, tokenIndexStart: number = state.tokenIndex): boolean {
+    return (
+        isTokenKind(state, Token.TokenKind.Identifier, tokenIndexStart) &&
+        isTokenKind(state, Token.TokenKind.Bang, tokenIndexStart + 1)
+    );
+}
+
 // Assumes a call to readPrimaryExpression has already happened.
 export function isRecursivePrimaryExpressionNext(
     state: ParseState,
     tokenIndexStart: number = state.tokenIndex,
 ): boolean {
     return (
         // section-access-expression
-        // this.isOnTokenKind(TokenKind.Bang)
+        // is detected up front by isSectionAccessExpressionNext instead of here
+        // because its `identifier ! identifier` rather than a suffix on an arbitrary primary-expression
         // field-access-expression
         isTokenKind(state, Token.TokenKind.LeftBrace, tokenIndexStart) ||
         // item-access-expression

src/powerquery-parser/parser/parser/parser.ts

@@ -167,6 +167,13 @@ export interface Parser {
         correlationId: number | undefined,
     ) => Ast.IdentifierExpression;
 
+    // 12.2.3.13 Section-access expression
+    readonly readSectionAccessExpression: (
+        state: ParseState,
+        parser: Parser,
+        correlationId: number | undefined,
+    ) => Ast.SectionAccessExpression;
+
     // 12.2.3.14 Parenthesized expression
     readonly readParenthesizedExpression: (
         state: ParseState,

src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts

@@ -92,6 +92,8 @@ export const CombinatorialParserV2: Parser = {
 
     readIdentifierExpression: NaiveParseSteps.readIdentifierExpression,
 
+    readSectionAccessExpression: NaiveParseSteps.readSectionAccessExpression,
+
     readParenthesizedExpression: NaiveParseSteps.readParenthesizedExpression,
 
     readNotImplementedExpression: NaiveParseSteps.readNotImplementedExpression,
@@ -205,7 +207,10 @@ async function readUnaryExpression(
         // PrimaryExpression
         case Token.TokenKind.AtSign:
         case Token.TokenKind.Identifier:
-            primaryExpression = NaiveParseSteps.readIdentifierExpression(state, parser, trace.id);
+            primaryExpression = ParseStateUtils.isSectionAccessExpressionNext(state)
+                ? NaiveParseSteps.readSectionAccessExpression(state, parser, trace.id)
+                : NaiveParseSteps.readIdentifierExpression(state, parser, trace.id);
+
             break;
 
         case Token.TokenKind.LeftParenthesis:

src/powerquery-parser/parser/parsers/naiveParseSteps.ts

@@ -898,7 +898,9 @@ export async function readPrimaryExpression(
     const isIdentifierExpressionNext: boolean =
         currentTokenKind === TokenKind.AtSign || currentTokenKind === TokenKind.Identifier;
 
-    if (isIdentifierExpressionNext) {
+    if (ParseStateUtils.isSectionAccessExpressionNext(state)) {
+        primaryExpression = parser.readSectionAccessExpression(state, parser, trace.id);
+    } else if (isIdentifierExpressionNext) {
         primaryExpression = parser.readIdentifierExpression(state, parser, trace.id);
     } else {
         switch (currentTokenKind) {
@@ -1201,6 +1203,63 @@ export function readIdentifierExpression(
     return identifierExpression;
 }
 
+// ---------------------------------------------------------
+// ---------- 12.2.3.13 Section-access expression ----------
+// ---------------------------------------------------------
+
+export function readSectionAccessExpression(
+    state: ParseState,
+    parser: Parser,
+    correlationId: number | undefined,
+): Ast.SectionAccessExpression {
+    const nodeKind: Ast.NodeKind.SectionAccessExpression = Ast.NodeKind.SectionAccessExpression;
+
+    const trace: Trace = state.traceManager.entry(
+        NaiveTraceConstant.Parse,
+        readSectionAccessExpression.name,
+        correlationId,
+        { [NaiveTraceConstant.TokenIndex]: state.tokenIndex },
+    );
+
+    state.cancellationToken?.throwIfCancelled();
+    ParseStateUtils.startContext(state, nodeKind);
+
+    const sectionIdentifier: Ast.Identifier = parser.readIdentifier(
+        state,
+        parser,
+        Ast.IdentifierContextKind.Value,
+        trace.id,
+    );
+
+    const bangConstant: Ast.IConstant<Constant.MiscConstant.Bang> = readTokenKindAsConstant(
+        state,
+        TokenKind.Bang,
+        Constant.MiscConstant.Bang,
+        trace.id,
+    );
+
+    const memberIdentifier: Ast.Identifier = parser.readIdentifier(
+        state,
+        parser,
+        Ast.IdentifierContextKind.Value,
+        trace.id,
+    );
+
+    const sectionAccessExpression: Ast.SectionAccessExpression = {
+        ...ParseStateUtils.assertGetContextNodeMetadata(state),
+        kind: nodeKind,
+        isLeaf: false,
+        sectionIdentifier,
+        bangConstant,
+        memberIdentifier,
+    };
+
+    ParseStateUtils.endContext(state, sectionAccessExpression);
+    trace.exit({ [NaiveTraceConstant.TokenIndex]: state.tokenIndex });
+
+    return sectionAccessExpression;
+}
+
 // --------------------------------------------------------
 // ---------- 12.2.3.14 Parenthesized expression ----------
 // --------------------------------------------------------

src/powerquery-parser/parser/parsers/recursiveDescentParser.ts

@@ -46,6 +46,8 @@ export const RecursiveDescentParser: Parser = {
 
     readIdentifierExpression: NaiveParseSteps.readIdentifierExpression,
 
+    readSectionAccessExpression: NaiveParseSteps.readSectionAccessExpression,
+
     readParenthesizedExpression: NaiveParseSteps.readParenthesizedExpression,
 
     readNotImplementedExpression: NaiveParseSteps.readNotImplementedExpression,