TokenStream.php 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281
  1. <?php declare(strict_types=1);
  2. namespace PhpParser\Internal;
  3. /**
  4. * Provides operations on token streams, for use by pretty printer.
  5. *
  6. * @internal
  7. */
  8. class TokenStream
  9. {
  10. /** @var array Tokens (in token_get_all format) */
  11. private $tokens;
  12. /** @var int[] Map from position to indentation */
  13. private $indentMap;
  14. /**
  15. * Create token stream instance.
  16. *
  17. * @param array $tokens Tokens in token_get_all() format
  18. */
  19. public function __construct(array $tokens) {
  20. $this->tokens = $tokens;
  21. $this->indentMap = $this->calcIndentMap();
  22. }
  23. /**
  24. * Whether the given position is immediately surrounded by parenthesis.
  25. *
  26. * @param int $startPos Start position
  27. * @param int $endPos End position
  28. *
  29. * @return bool
  30. */
  31. public function haveParens(int $startPos, int $endPos) : bool {
  32. return $this->haveTokenImmediatelyBefore($startPos, '(')
  33. && $this->haveTokenImmediatelyAfter($endPos, ')');
  34. }
  35. /**
  36. * Whether the given position is immediately surrounded by braces.
  37. *
  38. * @param int $startPos Start position
  39. * @param int $endPos End position
  40. *
  41. * @return bool
  42. */
  43. public function haveBraces(int $startPos, int $endPos) : bool {
  44. return ($this->haveTokenImmediatelyBefore($startPos, '{')
  45. || $this->haveTokenImmediatelyBefore($startPos, T_CURLY_OPEN))
  46. && $this->haveTokenImmediatelyAfter($endPos, '}');
  47. }
  48. /**
  49. * Check whether the position is directly preceded by a certain token type.
  50. *
  51. * During this check whitespace and comments are skipped.
  52. *
  53. * @param int $pos Position before which the token should occur
  54. * @param int|string $expectedTokenType Token to check for
  55. *
  56. * @return bool Whether the expected token was found
  57. */
  58. public function haveTokenImmediatelyBefore(int $pos, $expectedTokenType) : bool {
  59. $tokens = $this->tokens;
  60. $pos--;
  61. for (; $pos >= 0; $pos--) {
  62. $tokenType = $tokens[$pos][0];
  63. if ($tokenType === $expectedTokenType) {
  64. return true;
  65. }
  66. if ($tokenType !== \T_WHITESPACE
  67. && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
  68. break;
  69. }
  70. }
  71. return false;
  72. }
  73. /**
  74. * Check whether the position is directly followed by a certain token type.
  75. *
  76. * During this check whitespace and comments are skipped.
  77. *
  78. * @param int $pos Position after which the token should occur
  79. * @param int|string $expectedTokenType Token to check for
  80. *
  81. * @return bool Whether the expected token was found
  82. */
  83. public function haveTokenImmediatelyAfter(int $pos, $expectedTokenType) : bool {
  84. $tokens = $this->tokens;
  85. $pos++;
  86. for (; $pos < \count($tokens); $pos++) {
  87. $tokenType = $tokens[$pos][0];
  88. if ($tokenType === $expectedTokenType) {
  89. return true;
  90. }
  91. if ($tokenType !== \T_WHITESPACE
  92. && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
  93. break;
  94. }
  95. }
  96. return false;
  97. }
  98. public function skipLeft(int $pos, $skipTokenType) {
  99. $tokens = $this->tokens;
  100. $pos = $this->skipLeftWhitespace($pos);
  101. if ($skipTokenType === \T_WHITESPACE) {
  102. return $pos;
  103. }
  104. if ($tokens[$pos][0] !== $skipTokenType) {
  105. // Shouldn't happen. The skip token MUST be there
  106. throw new \Exception('Encountered unexpected token');
  107. }
  108. $pos--;
  109. return $this->skipLeftWhitespace($pos);
  110. }
  111. public function skipRight(int $pos, $skipTokenType) {
  112. $tokens = $this->tokens;
  113. $pos = $this->skipRightWhitespace($pos);
  114. if ($skipTokenType === \T_WHITESPACE) {
  115. return $pos;
  116. }
  117. if ($tokens[$pos][0] !== $skipTokenType) {
  118. // Shouldn't happen. The skip token MUST be there
  119. throw new \Exception('Encountered unexpected token');
  120. }
  121. $pos++;
  122. return $this->skipRightWhitespace($pos);
  123. }
  124. /**
  125. * Return first non-whitespace token position smaller or equal to passed position.
  126. *
  127. * @param int $pos Token position
  128. * @return int Non-whitespace token position
  129. */
  130. public function skipLeftWhitespace(int $pos) {
  131. $tokens = $this->tokens;
  132. for (; $pos >= 0; $pos--) {
  133. $type = $tokens[$pos][0];
  134. if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
  135. break;
  136. }
  137. }
  138. return $pos;
  139. }
  140. /**
  141. * Return first non-whitespace position greater or equal to passed position.
  142. *
  143. * @param int $pos Token position
  144. * @return int Non-whitespace token position
  145. */
  146. public function skipRightWhitespace(int $pos) {
  147. $tokens = $this->tokens;
  148. for ($count = \count($tokens); $pos < $count; $pos++) {
  149. $type = $tokens[$pos][0];
  150. if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
  151. break;
  152. }
  153. }
  154. return $pos;
  155. }
  156. public function findRight(int $pos, $findTokenType) {
  157. $tokens = $this->tokens;
  158. for ($count = \count($tokens); $pos < $count; $pos++) {
  159. $type = $tokens[$pos][0];
  160. if ($type === $findTokenType) {
  161. return $pos;
  162. }
  163. }
  164. return -1;
  165. }
  166. /**
  167. * Whether the given position range contains a certain token type.
  168. *
  169. * @param int $startPos Starting position (inclusive)
  170. * @param int $endPos Ending position (exclusive)
  171. * @param int|string $tokenType Token type to look for
  172. * @return bool Whether the token occurs in the given range
  173. */
  174. public function haveTokenInRange(int $startPos, int $endPos, $tokenType) {
  175. $tokens = $this->tokens;
  176. for ($pos = $startPos; $pos < $endPos; $pos++) {
  177. if ($tokens[$pos][0] === $tokenType) {
  178. return true;
  179. }
  180. }
  181. return false;
  182. }
  183. public function haveBracesInRange(int $startPos, int $endPos) {
  184. return $this->haveTokenInRange($startPos, $endPos, '{')
  185. || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
  186. || $this->haveTokenInRange($startPos, $endPos, '}');
  187. }
  188. /**
  189. * Get indentation before token position.
  190. *
  191. * @param int $pos Token position
  192. *
  193. * @return int Indentation depth (in spaces)
  194. */
  195. public function getIndentationBefore(int $pos) : int {
  196. return $this->indentMap[$pos];
  197. }
  198. /**
  199. * Get the code corresponding to a token offset range, optionally adjusted for indentation.
  200. *
  201. * @param int $from Token start position (inclusive)
  202. * @param int $to Token end position (exclusive)
  203. * @param int $indent By how much the code should be indented (can be negative as well)
  204. *
  205. * @return string Code corresponding to token range, adjusted for indentation
  206. */
  207. public function getTokenCode(int $from, int $to, int $indent) : string {
  208. $tokens = $this->tokens;
  209. $result = '';
  210. for ($pos = $from; $pos < $to; $pos++) {
  211. $token = $tokens[$pos];
  212. if (\is_array($token)) {
  213. $type = $token[0];
  214. $content = $token[1];
  215. if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
  216. $result .= $content;
  217. } else {
  218. // TODO Handle non-space indentation
  219. if ($indent < 0) {
  220. $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
  221. } elseif ($indent > 0) {
  222. $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
  223. } else {
  224. $result .= $content;
  225. }
  226. }
  227. } else {
  228. $result .= $token;
  229. }
  230. }
  231. return $result;
  232. }
  233. /**
  234. * Precalculate the indentation at every token position.
  235. *
  236. * @return int[] Token position to indentation map
  237. */
  238. private function calcIndentMap() {
  239. $indentMap = [];
  240. $indent = 0;
  241. foreach ($this->tokens as $token) {
  242. $indentMap[] = $indent;
  243. if ($token[0] === \T_WHITESPACE) {
  244. $content = $token[1];
  245. $newlinePos = \strrpos($content, "\n");
  246. if (false !== $newlinePos) {
  247. $indent = \strlen($content) - $newlinePos - 1;
  248. }
  249. }
  250. }
  251. // Add a sentinel for one past end of the file
  252. $indentMap[] = $indent;
  253. return $indentMap;
  254. }
  255. }