ScannedTokens.hpp

Go to the documentation of this file.

// @formatter:off
//
// Balau core C++ library
//
// Copyright (C) 2008 Bora Software (contact@borasoftware.com)
//
// Licensed under the Boost Software License - Version 1.0 - August 17th, 2003.
// See the LICENSE file for the full license text.
//


#ifndef COM_BORA_SOFTWARE__BALAU_LANG__SCANNED_TOKENS
#define COM_BORA_SOFTWARE__BALAU_LANG__SCANNED_TOKENS

#include <Balau/Exception/ContainerExceptions.hpp>
#include <Balau/Exception/ParsingExceptions.hpp>
#include <Balau/Lang/Common/CodeSpan.hpp>
#include <Balau/Dev/Assert.hpp>
#include <Balau/Util/Strings.hpp>

#include <stack>

#pragma clang diagnostic push
#pragma ide diagnostic ignored "OCUnusedGlobalDeclarationInspection"

namespace Balau::Lang {

enum class WhitespaceMode {
      ConsumeWhitespaceAndComments
    , ConsumeWhitespace
    , ConsumeBlanksAndComments
    , ConsumeBlanks
    , ConsumeLineBreaksAndComments
    , ConsumeLineBreaks
    , ConsumeComments
    , DoNotConsume
};

template <typename TokenT> class ScannedTokens;
template <typename TokenT> class RandomAccessScannedTokens;
template <typename TokenT> class ScannerApiScannedTokens;
template <typename TokenT> class IterativeScannedTokens;

template <typename TokenT> class ScannedTokens final {
    public: ScannedTokens(std::shared_ptr<const Resource::Uri> uri_,
                          std::string && text_,
                          std::vector<TokenT> && tokens_,
                          std::vector<unsigned int> && startOffsets_)
        : uri(std::move(uri_))
        , text(std::move(text_))
        , tokens(tokens_)
        , startOffsets(startOffsets_) {
        Assert::assertion(tokens.back() == TokenT::EndOfFile, "ScannedTokens must end with an EndOfFile token.");
    }

    public: std::string_view getText(unsigned int index) {
        if (index >= tokens.size()) {
            return std::string_view(text.data() + text.length(), 0);
        } else if (index == tokens.size() - 1) {
            return std::string_view(
                text.data() + startOffsets[index], text.length() - startOffsets[index]
            );
        } else {
            return std::string_view(
                  text.data() + startOffsets[index]
                , startOffsets[index + 1] - startOffsets[index]
            );
        }
    }

    public: std::string && moveTextOut() {
        return std::move(text);
    }

    public: std::shared_ptr<const Resource::Uri> getUri() {
        return uri;
    }


    private: std::shared_ptr<const Resource::Uri> uri;
    private: std::string text;
    private: std::vector<TokenT> tokens;
    private: std::vector<unsigned int> startOffsets;

    friend class ScannerApiScannedTokens<TokenT>;
    friend class IterativeScannedTokens<TokenT>;
    friend class RandomAccessScannedTokens<TokenT>;
};


template <typename TokenT> class RandomAccessScannedTokens final {
    public: explicit RandomAccessScannedTokens(ScannedTokens<TokenT> && scannedTokens_)
        : scannedTokens(std::move(scannedTokens_))
        , codeSpans(buildCodeSpans(scannedTokens)) {}

    public: const ScannedTokens<TokenT> & getScannedTokens() const {
        return scannedTokens;
    }

    public: const std::vector<CodeSpan> & getCodeSpans() const {
        return codeSpans;
    }

    public: static CodeSpan determineCodeSpan(const ScannedTokens<TokenT> & scannedTokens,
                                              unsigned int index) {
        return determineCodeSpan(scannedTokens.text, scannedTokens.tokens, scannedTokens.startOffsets, index);
    }

    public: static CodeSpan determineCodeSpan(const std::string & text,
                                              const std::vector<TokenT> & tokens,
                                              const std::vector<unsigned int> & startOffsets,
                                              unsigned int index) {
        if (tokens.size() <= index) {
            ThrowBalauException(
                  Exception::IndexOutOfRangeException
                , std::string("scanned token range is 0 to ") + ::toString(tokens.size()), index
            );
        }

        std::vector<unsigned int> lineLengthCache;
        CodePosition startPosition(1, 1);
        CodePosition endPosition(1, 1);

        // tokens.size() > 0

        for (size_t thisIndex = 0; thisIndex < tokens.size() - 1; thisIndex++) {
            determineNewEnd(
                  tokens[thisIndex]
                , text
                , startOffsets[thisIndex]
                , startOffsets[thisIndex + 1]
                , endPosition
                , lineLengthCache
            );

            startPosition = endPosition;

            if (index == thisIndex) {
                return CodeSpan(startPosition, endPosition);
            }
        }

        determineNewEnd(
              tokens[tokens.size() - 1]
            , text
            , startOffsets[tokens.size() - 1]
            , (unsigned int) text.length()
            , endPosition
            , lineLengthCache
        );

        return CodeSpan(startPosition, endPosition);
    }

    public: std::string && moveTextOut() {
        return std::move(scannedTokens.moveTextOut());
    }


    // Builds all the code spans from the supplied scanned tokens.
    private: static std::vector<CodeSpan> buildCodeSpans(const ScannedTokens<TokenT> & scannedTokens) {
        const std::vector<TokenT> & tokens = scannedTokens.tokens;
        const std::vector<unsigned int> & startOffsets = scannedTokens.startOffsets;

        std::vector<CodeSpan> builtCodeSpans;

        if (tokens.size() == 0) {
            return builtCodeSpans;
        }

        std::vector<unsigned int> lineLengthCache;
        CodePosition startPosition(1, 1);
        CodePosition endPosition(1, 1);

        // tokens.size() > 0

        for (size_t index = 0; index < tokens.size() - 1; index++) {
            determineNewEnd(
                  tokens[index]
                , scannedTokens.text
                , startOffsets[index]
                , startOffsets[index + 1]
                , endPosition
                , lineLengthCache
            );

            builtCodeSpans.emplace_back(CodeSpan(startPosition, endPosition));

            startPosition = endPosition;
        }

        determineNewEnd(
              tokens[tokens.size() - 1]
            , scannedTokens.text
            , startOffsets[tokens.size() - 1]
            , (unsigned int) scannedTokens.text.length()
            , endPosition
            , lineLengthCache
        );

        builtCodeSpans.emplace_back(CodeSpan(startPosition, endPosition));

        return builtCodeSpans;
    }

    // Updates in place the supplied end position for advancing.
    // Also updates the line length cache when a line break is encountered.
    private: static void determineNewEnd(TokenT token,
                                         const std::string & fullText,
                                         unsigned int thisOffset,
                                         unsigned int nextOffset,
                                         CodePosition & endPosition,
                                         std::vector<unsigned int> & lineLengthCache) {
        static const std::regex endLineBreakRegex = std::regex("(\n\r|\r\n|\n|\r)$");

        if (token == TokenT::LineBreak || token == TokenT::CommentLine || token == TokenT::CommentBlock) {
            const std::string thisText = fullText.substr(thisOffset, nextOffset - thisOffset);
            const std::vector<size_t> lineLengths = Util::Strings::lineLengths(thisText);
            const bool hasCurrentLine = !std::regex_search(thisText, endLineBreakRegex);

            using ::toString;

            Assert::assertion(
                  !lineLengths.empty()
                , [&] () { return std::string(
                    "A line break token was supplied in the determineNewEnd function"
                    "call but no line end was found in the supplied text. "
                    "Start offset = "
                ) + toString(thisOffset) + ", end offset = " +
                    toString(nextOffset) + ", token text = " + thisText; }
            );

            // The first line length is added to the current line length, then
            // the intermediate line lengths are added, then the current line
            // length becomes the current line length.

            // First.
            endPosition.column += lineLengths[0];
            ++endPosition.line;
            lineLengthCache.push_back(endPosition.column - 1); // one indexed column position

            if (hasCurrentLine) {
                // Intermediate.
                for (size_t m = 1; m < lineLengths.size() - 1; m++) {
                    lineLengthCache.push_back((unsigned int) lineLengths[m]);
                    ++endPosition.line;
                }

                // Current.
                endPosition.column = 1 + (unsigned int) lineLengths.back();
            } else {
                // Intermediate.
                for (size_t m = 1; m < lineLengths.size(); m++) {
                    lineLengthCache.push_back((unsigned int) lineLengths[m]);
                    ++endPosition.line;
                }

                // Current.
                endPosition.column = 1;
            }
        } else {
            endPosition.column += nextOffset - thisOffset;
        }
    }

    // Updates in place the supplied start position when retreating.
    // Also updates the line length cache when a line break is encountered.
    private: static void determineNewStart(TokenT token,
                                           const std::string & fullText,
                                           unsigned int thisOffset,
                                           unsigned int nextOffset,
                                           CodePosition & startPosition,
                                           std::vector<unsigned int> & lineLengthCache) {
        static const std::regex lineBreakRegex = std::regex("\n\r|\r\n|\n|\r");

        if (token == TokenT::LineBreak || token == TokenT::CommentLine || token == TokenT::CommentBlock) {
            const unsigned int lineBreaks = (unsigned int) Util::Strings::occurrences(fullText.substr(thisOffset, nextOffset - thisOffset), lineBreakRegex);

            using ::toString;

            Assert::assertion(
                  lineBreaks != 0
                , [&] () {
                    return toString(
                          "A line break token was supplied in the determineNewStart function"
                        , "call but no line end was found in the supplied text. "
                        , "Start offset = ", thisOffset
                        , ", end offset = ", nextOffset
                        , ", token text = ", fullText.substr(thisOffset, nextOffset - thisOffset)
                    );
                }
            );

            for (size_t m = 0; m < lineBreaks; m++) {
                startPosition.column = lineLengthCache.back() + 1; // one indexed column position
                lineLengthCache.pop_back();
            }

            startPosition.line -= lineBreaks;
        } else {
            startPosition.column -= nextOffset - thisOffset;
        }
    }

    friend class ScannerApiScannedTokens<TokenT>;

    private: ScannedTokens<TokenT> scannedTokens;
    private: std::vector<CodeSpan> codeSpans;
};


template <typename TokenT> class ScannedToken {
    public: TokenT token;

    public: CodeSpan codeSpan;

    public: std::string_view text;


    private: ScannedToken(TokenT token_, CodeSpan codeSpan_, std::string_view text_)
        : token(token_)
        , codeSpan(std::move(codeSpan_))
        , text(text_) {}

    friend class ScannerApiScannedTokens<TokenT>;
};

template <typename TokenT> class ScannerApiScannedTokens final {
    public: explicit ScannerApiScannedTokens(ScannedTokens<TokenT> && scannedTokens_) noexcept
        : scannedTokens(std::move(scannedTokens_))
        , currentIndex(0)
        , currentCodeSpan(1, 1, 1, 1) {
        whitespaceModeStack.push(WhitespaceMode::DoNotConsume);
        if (scannedTokens.tokens.size() > 1) {
            RandomAccessScannedTokens<TokenT>::determineNewEnd(
                  scannedTokens.tokens[0]
                , scannedTokens.text
                , scannedTokens.startOffsets[0]
                , scannedTokens.startOffsets[1]
                , currentCodeSpan.end
                , lineLengthCache
            );
        }
    }

    public: void reset() {
        while (!whitespaceModeStack.empty()) {
            whitespaceModeStack.pop();
        }

        whitespaceModeStack.push(WhitespaceMode::DoNotConsume);
        currentIndex = 0;
        currentCodeSpan = CodeSpan(1, 1, 1, 1);
        lineLengthCache.clear();
    }

    public: ScannedToken<TokenT> get() {
        const std::vector<TokenT> & tokens = scannedTokens.tokens;

        switch (whitespaceModeStack.top()) {
            case WhitespaceMode::ConsumeWhitespaceAndComments: {
                while (tokens[currentIndex] == TokenT::Blank
                    || tokens[currentIndex] == TokenT::LineBreak
                    || tokens[currentIndex] == TokenT::CommentLine) {
                    advanceCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeWhitespace: {
                while (tokens[currentIndex] == TokenT::Blank || tokens[currentIndex] == TokenT::LineBreak) {
                    advanceCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeBlanksAndComments: {
                while (tokens[currentIndex] == TokenT::Blank || tokens[currentIndex] == TokenT::CommentLine) {
                    advanceCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeBlanks: {
                while (tokens[currentIndex] == TokenT::Blank) {
                    advanceCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeLineBreaksAndComments: {
                while (tokens[currentIndex] == TokenT::LineBreak || tokens[currentIndex] == TokenT::CommentLine) {
                    advanceCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeLineBreaks: {
                while (tokens[currentIndex] == TokenT::LineBreak) {
                    advanceCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeComments: {
                while (tokens[currentIndex] == TokenT::CommentLine) {
                    advanceCurrentIndex();
                }

                break;
            }

            default: { // WhitespaceMode::DoNotConsume
                break;
            }
        }

        return ScannedToken<TokenT>(
            tokens[currentIndex], currentCodeSpan, scannedTokens.getText(currentIndex)
        );
    }

    public: void consume() {
        advanceCurrentIndex();
    }

    public: void expect(const TokenT token, std::string_view errorMessage) {
        get();

        if (scannedTokens.tokens[currentIndex] == token) {
            if (currentIndex < scannedTokens.tokens.size() - 1) {
                advanceCurrentIndex();
            }
        } else {
            ThrowBalauException(Exception::SyntaxErrorException, std::string(errorMessage), getCurrentCodeSpan());
        }
    }

    public: template <template <typename ...> class ContainerT, typename ... ArgT, typename ReportT>
    bool expect(const TokenT token,
                ContainerT<ReportT, ArgT ...> & container,
                const size_t maxErrorCount,
                const std::function<ReportT (const TokenT &, const CodeSpan &, size_t)> & errorReport) {
        get();

        if (scannedTokens.tokens[currentIndex] == token) {
            if (currentIndex < scannedTokens.tokens.size() - 1) {
                advanceCurrentIndex();
            }

            return true;
        } else {
            auto error = errorReport(token, getCurrentCodeSpan(), container.size());

            if (container.size() < maxErrorCount) {
                container.push_back(errorReport(token, getCurrentCodeSpan(), container.size()));
            }

            return false;
        }
    }

    public: void expect(const std::vector<TokenT> & tokens, std::string_view errorMessage) {
        get();

        if (std::find(tokens.begin(), tokens.end(), scannedTokens.tokens[currentIndex]) != tokens.end()) {
            advanceCurrentIndex();
        } else {
            ThrowBalauException(Exception::SyntaxErrorException, std::string(errorMessage), getCurrentCodeSpan());
        }
    }

    public: template <template <typename ...> class TokenContainerT, template <typename ...> class ContainerT, typename ... TokenArgT, typename ... ArgT, typename ReportT>
    bool expect(const TokenContainerT<TokenT, TokenArgT ...> & tokens,
                ContainerT<ReportT, ArgT ...> & container,
                const size_t maxErrorCount,
                const std::function<ReportT (const TokenContainerT<TokenT, TokenArgT ...> &, const CodeSpan &, size_t)> & errorReport) {
        get();

        if (std::find(tokens.begin(), tokens.end(), scannedTokens.tokens[currentIndex]) != tokens.end()) {
            advanceCurrentIndex();
            return true;
        } else {
            if (container.size() < maxErrorCount) {
                container.push_back(errorReport(tokens, getCurrentCodeSpan(), container.size()));
            }

            return false;
        }
    }

    public: class Marker {
        public: Marker(const Marker & rhs) noexcept : index(rhs.index) {}

        public: Marker(Marker && rhs) noexcept : index(rhs.index) {}

        public: Marker & operator = (const Marker & rhs) noexcept {
            index = rhs.index;
            return *this;
        }

        public: Marker & operator = (Marker && rhs) noexcept {
            index = rhs.index;
            return *this;
        }

        private: explicit Marker(size_t index_) : index(index_) {}

        private: size_t index;

        friend class ScannerApiScannedTokens;
    };

    public: Marker mark() const {
        return Marker(currentIndex);
    }

    public: void putBack(const Marker & marker) {
        if (marker.index > currentIndex) {
            ThrowBalauException(
                  Exception::IndexOutOfRangeException
                , ::toString("Marker index (", marker.index, ") is greater than current index (", currentIndex, ")")
                , marker.index
            );
        }

        const auto steps = currentIndex - marker.index;

        for (size_t m = 0; m < steps; m++) {
            putBack();
        }
    }

    public: void putBack() {
        const std::vector<TokenT> & tokens = scannedTokens.tokens;

        retreatCurrentIndex();

        switch (whitespaceModeStack.top()) {
            case WhitespaceMode::ConsumeWhitespaceAndComments: {
                while (currentIndex > 0 && (tokens[currentIndex - 1] == TokenT::Blank || tokens[currentIndex - 1] == TokenT::LineBreak || tokens[currentIndex - 1] == TokenT::CommentLine)) {
                    retreatCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeWhitespace: {
                while (currentIndex > 0 && (tokens[currentIndex - 1] == TokenT::Blank || tokens[currentIndex - 1] == TokenT::LineBreak)) {
                    retreatCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeBlanksAndComments: {
                while (currentIndex > 0 && (tokens[currentIndex - 1] == TokenT::Blank || tokens[currentIndex - 1] == TokenT::CommentLine)) {
                    retreatCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeBlanks: {
                while (currentIndex > 0 && tokens[currentIndex - 1] == TokenT::Blank) {
                    retreatCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeLineBreaksAndComments: {
                while (currentIndex > 0 && (tokens[currentIndex - 1] == TokenT::LineBreak || tokens[currentIndex - 1] == TokenT::CommentLine)) {
                    retreatCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeLineBreaks: {
                while (currentIndex > 0 && (tokens[currentIndex - 1] == TokenT::LineBreak)) {
                    retreatCurrentIndex();
                }

                break;
            }

            case WhitespaceMode::ConsumeComments: {
                while (currentIndex > 0 && (tokens[currentIndex - 1] == TokenT::CommentLine)) {
                    retreatCurrentIndex();
                }

                break;
            }

            default: { // WhitespaceMode::DoNotConsume
                break;
            }
        }
    }

    public: CodeSpan getCurrentCodeSpan() const {
        return currentCodeSpan;
    }

    public: bool currentIsBlank() const {
        return currentIndex < scannedTokens.tokens.size() && scannedTokens.tokens[currentIndex] == TokenT::Blank;
    }

    public: bool currentIsLineBreak() const {
        return currentIndex < scannedTokens.tokens.size() && scannedTokens.tokens[currentIndex] == TokenT::LineBreak;
    }

    public: bool currentIsWhitespace() const {
        if (currentIndex >= scannedTokens.tokens.size()) {
            return false;
        }

        const TokenT token = scannedTokens.tokens[currentIndex];
        return token == TokenT::Blank || token == TokenT::LineBreak;
    }

    public: bool isBlank(const Marker & marker) const {
        return scannedTokens.tokens[marker.index] == TokenT::Blank;
    }

    public: bool isLineBreak(const Marker & marker) const {
        return scannedTokens.tokens[marker.index] == TokenT::LineBreak;
    }

    public: bool isWhitespace(const Marker & marker) const {
        const TokenT token = scannedTokens.tokens[marker.index];
        return token == TokenT::Blank || token == TokenT::LineBreak;
    }

    public: void pushWhitespaceMode(WhitespaceMode newMode) {
        whitespaceModeStack.push(newMode);
    }

    public: void popWhitespaceMode() {
        whitespaceModeStack.pop();
    }

    public: size_t size() const {
        return scannedTokens.tokens.size();
    }

    public: std::string && moveTextOut() {
        return std::move(scannedTokens.moveTextOut());
    }

    public: std::shared_ptr<const Resource::Uri> getUri() {
        return scannedTokens.getUri();
    }


    private: void advanceCurrentIndex() {
        if (currentIndex == scannedTokens.tokens.size() - 1) {
            return;
        }

        currentCodeSpan.start = currentCodeSpan.end;
        ++currentIndex;

        if (currentIndex < scannedTokens.tokens.size() - 1) {
            // Calculate the new end point.
            RandomAccessScannedTokens<TokenT>::determineNewEnd(
                  scannedTokens.tokens[currentIndex]
                , scannedTokens.text
                , scannedTokens.startOffsets[currentIndex]
                , scannedTokens.startOffsets[currentIndex + 1]
                , currentCodeSpan.end
                , lineLengthCache
            );
        }
    }

    private: void retreatCurrentIndex() {
        if (currentIndex == 0) {
            return;
        }

        currentCodeSpan.end = currentCodeSpan.start;

        RandomAccessScannedTokens<TokenT>::determineNewStart(
              scannedTokens.tokens[currentIndex - 1]
            , scannedTokens.text
            , scannedTokens.startOffsets[currentIndex - 1]
            , scannedTokens.startOffsets[currentIndex]
            , currentCodeSpan.start
            , lineLengthCache
        );

        --currentIndex;
    }

    private: ScannedTokens<TokenT> scannedTokens;
    private: unsigned int currentIndex;
    private: CodeSpan currentCodeSpan;
    private: std::stack<WhitespaceMode> whitespaceModeStack;
    private: std::vector<unsigned int> lineLengthCache;
};


template <typename TokenT> class IterativeScannedTokens final {
    public: class ConstElement {
        private: const IterativeScannedTokens<TokenT> * parent;
        private: const size_t index;

        public: ConstElement(const ScannedTokens<TokenT> * parent_, size_t index_)
            : parent(parent_)
            , index(index_) {}

        public: TokenT token() const {
            return parent->getToken(index);
        }

        public: unsigned int startOffset() const {
            return parent->getStartOffset(index);
        }
    };

    public: class const_iterator {
        friend class IterativeScannedTokens<TokenT>;

        private: const IterativeScannedTokens<TokenT> * parent;
        private: size_t index;

        private: explicit const_iterator(const ScannedTokens<TokenT> * parent_, size_t index_)
            : parent(parent_)
            , index(index_) {}

        public: const_iterator(const const_iterator & copy)
            : parent(copy.parent)
            , index(copy.index) {}

        public: const_iterator & operator = (const const_iterator & copy) {
            parent = copy.parent;
            index = copy.index;
            return *this;
        }

        public: const_iterator & operator ++ () {
            ++index;
            return *this;
        }

        public: ConstElement operator * () const {
            return ConstElement(parent->scannedTokens.tokens[index], parent->scannedTokens.startOffsets[index]);
        }

        public: ConstElement operator -> () const {
            return ConstElement(parent->scannedTokens.tokens[index], parent->scannedTokens.startOffsets[index]);
        }

        public: bool operator == (const const_iterator & rhs) const {
            return parent == rhs.parent && index == rhs.index;
        }

        public: bool operator != (const const_iterator & rhs) const {
            return ! operator == (rhs);
        }
    };

    public: class Element {
        private: IterativeScannedTokens<TokenT> * parent;
        private: const size_t index;

        public: Element(ScannedTokens<TokenT> * parent_, size_t index_)
            : parent(parent_)
            , index(index_) {}

        public: TokenT token() const {
            return parent->getToken(index);
        }

        public: TokenT & token() {
            return parent->getToken(index);
        }

        public: unsigned int startOffset() const {
            return parent->getStartOffset(index);
        }

        public: unsigned int & startOffset() {
            return parent->getStartOffset(index);
        }
    };

    public: class iterator {
        friend class IterativeScannedTokens<TokenT>;

        private: IterativeScannedTokens<TokenT> * parent;
        private: size_t index;

        private: explicit iterator(ScannedTokens<TokenT> * parent_, size_t index_)
            : parent(parent_)
            , index(index_) {}

        public: iterator(const iterator & copy)
            : parent(copy.parent)
            , index(copy.index) {}

        public: iterator & operator = (const iterator & copy) {
            parent = copy.parent;
            index = copy.index;
            return *this;
        }

        public: iterator & operator ++ () {
            ++index;
            return *this;
        }

        public: ConstElement operator * () const {
            return ConstElement(parent->scannedTokens.tokens[index], parent->scannedTokens.startOffsets[index]);
        }

        public: Element operator * () {
            return Element(parent->scannedTokens.tokens[index], parent->scannedTokens.startOffsets[index]);
        }

        public: ConstElement operator -> () const {
            return ConstElement(parent->scannedTokens.tokens[index], parent->scannedTokens.startOffsets[index]);
        }

        public: Element operator -> () {
            return Element(parent->scannedTokens.tokens[index], parent->scannedTokens.startOffsets[index]);
        }

        public: bool operator == (const iterator & rhs) const {
            return parent == rhs.parent && index == rhs.index;
        }

        public: bool operator != (const iterator & rhs) const {
            return ! operator == (rhs);
        }
    };

    friend class const_iterator;
    friend class iterator;

    public: explicit IterativeScannedTokens(ScannedTokens<TokenT> & scannedTokens_)
        : scannedTokens(scannedTokens_)  {}

    public: void addToken(TokenT token, unsigned int startOffset) {
        scannedTokens.tokens.emplace_back(token);
        scannedTokens.startOffsets.emplace_back(startOffset);
    }

    public: size_t size() const {
        return scannedTokens.tokens.size();
    }

    public: TokenT getToken(unsigned int index) const {
        return scannedTokens.tokens[index];
    }

    public: TokenT & getToken(unsigned int index) {
        return scannedTokens.tokens[index];
    }


    public: unsigned int getStartOffset(unsigned int index) const {
        return scannedTokens.startOffsets[index];
    }

    public: unsigned int & getStartOffset(unsigned int index) {
        return scannedTokens.startOffsets[index];
    }

    public: const std::vector<TokenT> & getTokens() const {
        return scannedTokens.tokens;
    }

    public: const std::vector<unsigned int> & getStartOffsets() const {
        return scannedTokens.startOffsets;
    }

    public: const_iterator begin() const {
        return const_iterator(this, 0);
    }

    public: iterator begin() {
        return iterator(this, 0);
    }

    public: const_iterator end() const {
        return const_iterator(this, scannedTokens.tokens.size());
    }

    public: iterator end() {
        return iterator(this, scannedTokens.tokens.size());
    }

    public: std::string && moveTextOut() {
        return std::move(scannedTokens.moveTextOut());
    }


    private: ScannedTokens<TokenT> scannedTokens;
};

} // namespace Balau::Lang

#pragma clang diagnostic pop

#endif // COM_BORA_SOFTWARE__BALAU_LANG__SCANNED_TOKENS