cuongpv/faveo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated-packages

Browse Source
This commit is contained in:
RafficMohammed
2023-01-08 00:13:22 +05:30
parent 3ff7df7487
commit da241bacb6
12659 changed files with 563377 additions and 510538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
vendor/nikic/php-parser/lib/PhpParser/Lexer.php vendored
View File

@@ -15,8 +15,15 @@ class Lexer
protected $tokenMap;
protected $dropTokens;
protected $identifierTokens;
protected $usedAttributes;
private $attributeStartLineUsed;
private $attributeEndLineUsed;
private $attributeStartTokenPosUsed;
private $attributeEndTokenPosUsed;
private $attributeStartFilePosUsed;
private $attributeEndFilePosUsed;
private $attributeCommentsUsed;
/**
* Creates a Lexer.
@@ -28,21 +35,28 @@ class Lexer
* first three. For more info see getNextToken() docs.
*/
public function __construct(array $options = []) {
// map from internal tokens to PhpParser tokens
// Create Map from internal tokens to PhpParser tokens.
$this->defineCompatibilityTokens();
$this->tokenMap = $this->createTokenMap();
$this->identifierTokens = $this->createIdentifierTokenMap();
// map of tokens to drop while lexing (the map is only used for isset lookup,
// that's why the value is simply set to 1; the value is never actually used.)
$this->dropTokens = array_fill_keys(
[\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
[\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
);
// the usedAttributes member is a map of the used attribute names to a dummy
// value (here "true")
$options += [
'usedAttributes' => ['comments', 'startLine', 'endLine'],
];
$this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
$defaultAttributes = ['comments', 'startLine', 'endLine'];
$usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
// Create individual boolean properties to make these checks faster.
$this->attributeStartLineUsed = isset($usedAttributes['startLine']);
$this->attributeEndLineUsed = isset($usedAttributes['endLine']);
$this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
$this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
$this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
$this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
$this->attributeCommentsUsed = isset($usedAttributes['comments']);
}
/**
@@ -71,9 +85,8 @@ class Lexer
$scream = ini_set('xdebug.scream', '0');
error_clear_last();
$this->tokens = @token_get_all($code);
$this->handleErrors($errorHandler);
$this->postprocessTokens($errorHandler);
if (false !== $scream) {
ini_set('xdebug.scream', $scream);
@@ -81,13 +94,9 @@ class Lexer
}
private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
$tokens = [];
for ($i = $start; $i < $end; $i++) {
$chr = $this->code[$i];
if ($chr === 'b' || $chr === 'B') {
// HHVM does not treat b" tokens correctly, so ignore these
continue;
}
if ($chr === "\0") {
// PHP cuts error message after null byte, so need special case
$errorMsg = 'Unexpected null byte';
@@ -97,6 +106,7 @@ class Lexer
);
}
$tokens[] = [\T_BAD_CHARACTER, $chr, $line];
$errorHandler->handleError(new Error($errorMsg, [
'startLine' => $line,
'endLine' => $line,
@@ -104,6 +114,7 @@ class Lexer
'endFilePos' => $i,
]));
}
return $tokens;
}
/**
@@ -117,43 +128,116 @@ class Lexer
&& substr($token[1], -2) !== '*/';
}
/**
* Check whether an error *may* have occurred during tokenization.
*
* @return bool
*/
private function errorMayHaveOccurred() : bool {
if (defined('HHVM_VERSION')) {
// In HHVM token_get_all() does not throw warnings, so we need to conservatively
// assume that an error occurred
return true;
}
return null !== error_get_last();
}
protected function handleErrors(ErrorHandler $errorHandler) {
if (!$this->errorMayHaveOccurred()) {
return;
}
protected function postprocessTokens(ErrorHandler $errorHandler) {
// PHP's error handling for token_get_all() is rather bad, so if we want detailed
// error information we need to compute it ourselves. Invalid character errors are
// detected by finding "gaps" in the token array. Unterminated comments are detected
// by checking if a trailing comment has a "*/" at the end.
//
// Additionally, we perform a number of canonicalizations here:
// * Use the PHP 8.0 comment format, which does not include trailing whitespace anymore.
// * Use PHP 8.0 T_NAME_* tokens.
// * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
// T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
$filePos = 0;
$line = 1;
foreach ($this->tokens as $token) {
$numTokens = \count($this->tokens);
for ($i = 0; $i < $numTokens; $i++) {
$token = $this->tokens[$i];
// Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
// In this case we only need to emit an error.
if ($token[0] === \T_BAD_CHARACTER) {
$this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
}
if ($token[0] === \T_COMMENT && substr($token[1], 0, 2) !== '/*'
&& preg_match('/(\r\n|\n|\r)$/D', $token[1], $matches)) {
$trailingNewline = $matches[0];
$token[1] = substr($token[1], 0, -strlen($trailingNewline));
$this->tokens[$i] = $token;
if (isset($this->tokens[$i + 1]) && $this->tokens[$i + 1][0] === \T_WHITESPACE) {
// Move trailing newline into following T_WHITESPACE token, if it already exists.
$this->tokens[$i + 1][1] = $trailingNewline . $this->tokens[$i + 1][1];
$this->tokens[$i + 1][2]--;
} else {
// Otherwise, we need to create a new T_WHITESPACE token.
array_splice($this->tokens, $i + 1, 0, [
[\T_WHITESPACE, $trailingNewline, $line],
]);
$numTokens++;
}
}
// Emulate PHP 8 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and T_STRING
// into a single token.
if (\is_array($token)
&& ($token[0] === \T_NS_SEPARATOR || isset($this->identifierTokens[$token[0]]))) {
$lastWasSeparator = $token[0] === \T_NS_SEPARATOR;
$text = $token[1];
for ($j = $i + 1; isset($this->tokens[$j]); $j++) {
if ($lastWasSeparator) {
if (!isset($this->identifierTokens[$this->tokens[$j][0]])) {
break;
}
$lastWasSeparator = false;
} else {
if ($this->tokens[$j][0] !== \T_NS_SEPARATOR) {
break;
}
$lastWasSeparator = true;
}
$text .= $this->tokens[$j][1];
}
if ($lastWasSeparator) {
// Trailing separator is not part of the name.
$j--;
$text = substr($text, 0, -1);
}
if ($j > $i + 1) {
if ($token[0] === \T_NS_SEPARATOR) {
$type = \T_NAME_FULLY_QUALIFIED;
} else if ($token[0] === \T_NAMESPACE) {
$type = \T_NAME_RELATIVE;
} else {
$type = \T_NAME_QUALIFIED;
}
$token = [$type, $text, $line];
array_splice($this->tokens, $i, $j - $i, [$token]);
$numTokens -= $j - $i - 1;
}
}
if ($token === '&') {
$next = $i + 1;
while (isset($this->tokens[$next]) && $this->tokens[$next][0] === \T_WHITESPACE) {
$next++;
}
$followedByVarOrVarArg = isset($this->tokens[$next]) &&
($this->tokens[$next][0] === \T_VARIABLE || $this->tokens[$next][0] === \T_ELLIPSIS);
$this->tokens[$i] = $token = [
$followedByVarOrVarArg
? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
: \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG,
'&',
$line,
];
}
$tokenValue = \is_string($token) ? $token : $token[1];
$tokenLen = \strlen($tokenValue);
if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
// Something is missing, must be an invalid character
$nextFilePos = strpos($this->code, $tokenValue, $filePos);
$this->handleInvalidCharacterRange(
$badCharTokens = $this->handleInvalidCharacterRange(
$filePos, $nextFilePos, $line, $errorHandler);
$filePos = (int) $nextFilePos;
array_splice($this->tokens, $i, 0, $badCharTokens);
$numTokens += \count($badCharTokens);
$i += \count($badCharTokens);
}
$filePos += $tokenLen;
@@ -176,8 +260,9 @@ class Lexer
$this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
} else {
// Invalid characters at the end of the input
$this->handleInvalidCharacterRange(
$badCharTokens = $this->handleInvalidCharacterRange(
$filePos, \strlen($this->code), $line, $errorHandler);
$this->tokens = array_merge($this->tokens, $badCharTokens);
}
return;
}
@@ -230,13 +315,13 @@ class Lexer
$token = "\0";
}
if (isset($this->usedAttributes['startLine'])) {
if ($this->attributeStartLineUsed) {
$startAttributes['startLine'] = $this->line;
}
if (isset($this->usedAttributes['startTokenPos'])) {
if ($this->attributeStartTokenPosUsed) {
$startAttributes['startTokenPos'] = $this->pos;
}
if (isset($this->usedAttributes['startFilePos'])) {
if ($this->attributeStartFilePosUsed) {
$startAttributes['startFilePos'] = $this->filePos;
}
@@ -254,7 +339,8 @@ class Lexer
$value = $token[1];
$id = $this->tokenMap[$token[0]];
if (\T_CLOSE_TAG === $token[0]) {
$this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
$this->prevCloseTagHasNewline = false !== strpos($token[1], "\n")
|| false !== strpos($token[1], "\r");
} elseif (\T_INLINE_HTML === $token[0]) {
$startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
}
@@ -262,27 +348,33 @@ class Lexer
$this->line += substr_count($value, "\n");
$this->filePos += \strlen($value);
} else {
$origLine = $this->line;
$origFilePos = $this->filePos;
$this->line += substr_count($token[1], "\n");
$this->filePos += \strlen($token[1]);
if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
if (isset($this->usedAttributes['comments'])) {
if ($this->attributeCommentsUsed) {
$comment = \T_DOC_COMMENT === $token[0]
? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
: new Comment($token[1], $this->line, $this->filePos, $this->pos);
? new Comment\Doc($token[1],
$origLine, $origFilePos, $this->pos,
$this->line, $this->filePos - 1, $this->pos)
: new Comment($token[1],
$origLine, $origFilePos, $this->pos,
$this->line, $this->filePos - 1, $this->pos);
$startAttributes['comments'][] = $comment;
}
}
$this->line += substr_count($token[1], "\n");
$this->filePos += \strlen($token[1]);
continue;
}
if (isset($this->usedAttributes['endLine'])) {
if ($this->attributeEndLineUsed) {
$endAttributes['endLine'] = $this->line;
}
if (isset($this->usedAttributes['endTokenPos'])) {
if ($this->attributeEndTokenPosUsed) {
$endAttributes['endTokenPos'] = $this->pos;
}
if (isset($this->usedAttributes['endFilePos'])) {
if ($this->attributeEndFilePosUsed) {
$endAttributes['endFilePos'] = $this->filePos - 1;
}
@@ -329,6 +421,66 @@ class Lexer
return substr($textAfter, strlen($matches[0]));
}
private function defineCompatibilityTokens() {
static $compatTokensDefined = false;
if ($compatTokensDefined) {
return;
}
$compatTokens = [
// PHP 7.4
'T_BAD_CHARACTER',
'T_FN',
'T_COALESCE_EQUAL',
// PHP 8.0
'T_NAME_QUALIFIED',
'T_NAME_FULLY_QUALIFIED',
'T_NAME_RELATIVE',
'T_MATCH',
'T_NULLSAFE_OBJECT_OPERATOR',
'T_ATTRIBUTE',
// PHP 8.1
'T_ENUM',
'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
'T_READONLY',
];
// PHP-Parser might be used together with another library that also emulates some or all
// of these tokens. Perform a sanity-check that all already defined tokens have been
// assigned a unique ID.
$usedTokenIds = [];
foreach ($compatTokens as $token) {
if (\defined($token)) {
$tokenId = \constant($token);
$clashingToken = $usedTokenIds[$tokenId] ?? null;
if ($clashingToken !== null) {
throw new \Error(sprintf(
'Token %s has same ID as token %s, ' .
'you may be using a library with broken token emulation',
$token, $clashingToken
));
}
$usedTokenIds[$tokenId] = $token;
}
}
// Now define any tokens that have not yet been emulated. Try to assign IDs from -1
// downwards, but skip any IDs that may already be in use.
$newTokenId = -1;
foreach ($compatTokens as $token) {
if (!\defined($token)) {
while (isset($usedTokenIds[$newTokenId])) {
$newTokenId--;
}
\define($token, $newTokenId);
$newTokenId--;
}
}
$compatTokensDefined = true;
}
/**
* Creates the token map.
*
@@ -373,6 +525,36 @@ class Lexer
$tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
}
// Assign tokens for which we define compatibility constants, as token_name() does not know them.
$tokenMap[\T_FN] = Tokens::T_FN;
$tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
$tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
$tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
$tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
$tokenMap[\T_MATCH] = Tokens::T_MATCH;
$tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
$tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
$tokenMap[\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
$tokenMap[\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG;
$tokenMap[\T_ENUM] = Tokens::T_ENUM;
$tokenMap[\T_READONLY] = Tokens::T_READONLY;
return $tokenMap;
}
private function createIdentifierTokenMap(): array {
// Based on semi_reserved production.
return array_fill_keys([
\T_STRING,
\T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
\T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
\T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
\T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
\T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
\T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
\T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
\T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
\T_MATCH,
], true);
}
}