Runner.php 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573
  1. <?php
  2. declare(strict_types=1);
  3. /*
  4. * This file is part of PHP CS Fixer.
  5. *
  6. * (c) Fabien Potencier <fabien@symfony.com>
  7. * Dariusz Rumiński <dariusz.ruminski@gmail.com>
  8. *
  9. * This source file is subject to the MIT license that is bundled
  10. * with this source code in the file LICENSE.
  11. */
  12. namespace PhpCsFixer\Runner;
  13. use Clue\React\NDJson\Decoder;
  14. use Clue\React\NDJson\Encoder;
  15. use PhpCsFixer\AbstractFixer;
  16. use PhpCsFixer\Cache\CacheManagerInterface;
  17. use PhpCsFixer\Cache\Directory;
  18. use PhpCsFixer\Cache\DirectoryInterface;
  19. use PhpCsFixer\Console\Command\WorkerCommand;
  20. use PhpCsFixer\Differ\DifferInterface;
  21. use PhpCsFixer\Error\Error;
  22. use PhpCsFixer\Error\ErrorsManager;
  23. use PhpCsFixer\Error\SourceExceptionFactory;
  24. use PhpCsFixer\FileReader;
  25. use PhpCsFixer\Fixer\FixerInterface;
  26. use PhpCsFixer\FixerFileProcessedEvent;
  27. use PhpCsFixer\Linter\LinterInterface;
  28. use PhpCsFixer\Linter\LintingException;
  29. use PhpCsFixer\Linter\LintingResultInterface;
  30. use PhpCsFixer\Preg;
  31. use PhpCsFixer\Runner\Parallel\ParallelAction;
  32. use PhpCsFixer\Runner\Parallel\ParallelConfig;
  33. use PhpCsFixer\Runner\Parallel\ParallelConfigFactory;
  34. use PhpCsFixer\Runner\Parallel\ParallelisationException;
  35. use PhpCsFixer\Runner\Parallel\ProcessFactory;
  36. use PhpCsFixer\Runner\Parallel\ProcessIdentifier;
  37. use PhpCsFixer\Runner\Parallel\ProcessPool;
  38. use PhpCsFixer\Runner\Parallel\WorkerException;
  39. use PhpCsFixer\Tokenizer\Tokens;
  40. use React\EventLoop\StreamSelectLoop;
  41. use React\Socket\ConnectionInterface;
  42. use React\Socket\TcpServer;
  43. use Symfony\Component\Console\Input\InputInterface;
  44. use Symfony\Component\EventDispatcher\EventDispatcherInterface;
  45. use Symfony\Component\Filesystem\Exception\IOException;
  46. use Symfony\Contracts\EventDispatcher\Event;
  47. /**
  48. * @author Dariusz Rumiński <dariusz.ruminski@gmail.com>
  49. * @author Greg Korba <greg@codito.dev>
  50. *
  51. * @phpstan-type _RunResult array<string, array{appliedFixers: list<string>, diff: string}>
  52. */
  53. final class Runner
  54. {
  55. private DifferInterface $differ;
  56. private ?DirectoryInterface $directory;
  57. private ?EventDispatcherInterface $eventDispatcher;
  58. private ErrorsManager $errorsManager;
  59. private CacheManagerInterface $cacheManager;
  60. private bool $isDryRun;
  61. private LinterInterface $linter;
  62. /**
  63. * @var null|\Traversable<array-key, \SplFileInfo>
  64. */
  65. private $fileIterator;
  66. private int $fileCount;
  67. /**
  68. * @var list<FixerInterface>
  69. */
  70. private array $fixers;
  71. private bool $stopOnViolation;
  72. private ParallelConfig $parallelConfig;
  73. private ?InputInterface $input;
  74. private ?string $configFile;
  75. /**
  76. * @param null|\Traversable<array-key, \SplFileInfo> $fileIterator
  77. * @param list<FixerInterface> $fixers
  78. */
  79. public function __construct(
  80. ?\Traversable $fileIterator,
  81. array $fixers,
  82. DifferInterface $differ,
  83. ?EventDispatcherInterface $eventDispatcher,
  84. ErrorsManager $errorsManager,
  85. LinterInterface $linter,
  86. bool $isDryRun,
  87. CacheManagerInterface $cacheManager,
  88. ?DirectoryInterface $directory = null,
  89. bool $stopOnViolation = false,
  90. // @TODO Make these arguments required in 4.0
  91. ?ParallelConfig $parallelConfig = null,
  92. ?InputInterface $input = null,
  93. ?string $configFile = null
  94. ) {
  95. // Required only for main process (calculating workers count)
  96. $this->fileCount = null !== $fileIterator ? \count(iterator_to_array($fileIterator)) : 0;
  97. $this->fileIterator = $fileIterator;
  98. $this->fixers = $fixers;
  99. $this->differ = $differ;
  100. $this->eventDispatcher = $eventDispatcher;
  101. $this->errorsManager = $errorsManager;
  102. $this->linter = $linter;
  103. $this->isDryRun = $isDryRun;
  104. $this->cacheManager = $cacheManager;
  105. $this->directory = $directory ?? new Directory('');
  106. $this->stopOnViolation = $stopOnViolation;
  107. $this->parallelConfig = $parallelConfig ?? ParallelConfigFactory::sequential();
  108. $this->input = $input;
  109. $this->configFile = $configFile;
  110. }
  111. /**
  112. * @TODO consider to drop this method and make iterator parameter obligatory in constructor,
  113. * more in https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/pull/7777/files#r1590447581
  114. *
  115. * @param \Traversable<array-key, \SplFileInfo> $fileIterator
  116. */
  117. public function setFileIterator(iterable $fileIterator): void
  118. {
  119. $this->fileIterator = $fileIterator;
  120. // Required only for main process (calculating workers count)
  121. $this->fileCount = \count(iterator_to_array($fileIterator));
  122. }
  123. /**
  124. * @return _RunResult
  125. */
  126. public function fix(): array
  127. {
  128. if (0 === $this->fileCount) {
  129. return [];
  130. }
  131. // @TODO Remove condition for the input argument in 4.0, as it should be required in the constructor
  132. return $this->parallelConfig->getMaxProcesses() > 1 && null !== $this->input
  133. ? $this->fixParallel()
  134. : $this->fixSequential();
  135. }
  136. /**
  137. * Heavily inspired by {@see https://github.com/phpstan/phpstan-src/blob/9ce425bca5337039fb52c0acf96a20a2b8ace490/src/Parallel/ParallelAnalyser.php}.
  138. *
  139. * @return _RunResult
  140. */
  141. private function fixParallel(): array
  142. {
  143. $changed = [];
  144. $streamSelectLoop = new StreamSelectLoop();
  145. $server = new TcpServer('127.0.0.1:0', $streamSelectLoop);
  146. $serverPort = parse_url($server->getAddress() ?? '', PHP_URL_PORT);
  147. if (!is_numeric($serverPort)) {
  148. throw new ParallelisationException(sprintf(
  149. 'Unable to parse server port from "%s"',
  150. $server->getAddress() ?? ''
  151. ));
  152. }
  153. $processPool = new ProcessPool($server);
  154. $maxFilesPerProcess = $this->parallelConfig->getFilesPerProcess();
  155. $fileIterator = $this->getFilteringFileIterator();
  156. $fileIterator->rewind();
  157. $getFileChunk = static function () use ($fileIterator, $maxFilesPerProcess): array {
  158. $files = [];
  159. while (\count($files) < $maxFilesPerProcess) {
  160. $current = $fileIterator->current();
  161. if (null === $current) {
  162. break;
  163. }
  164. $files[] = $current->getRealPath();
  165. $fileIterator->next();
  166. }
  167. return $files;
  168. };
  169. // [REACT] Handle worker's handshake (init connection)
  170. $server->on('connection', static function (ConnectionInterface $connection) use ($processPool, $getFileChunk): void {
  171. $jsonInvalidUtf8Ignore = \defined('JSON_INVALID_UTF8_IGNORE') ? JSON_INVALID_UTF8_IGNORE : 0;
  172. $decoder = new Decoder($connection, true, 512, $jsonInvalidUtf8Ignore);
  173. $encoder = new Encoder($connection, $jsonInvalidUtf8Ignore);
  174. // [REACT] Bind connection when worker's process requests "hello" action (enables 2-way communication)
  175. $decoder->on('data', static function (array $data) use ($processPool, $getFileChunk, $decoder, $encoder): void {
  176. if (ParallelAction::WORKER_HELLO !== $data['action']) {
  177. return;
  178. }
  179. $identifier = ProcessIdentifier::fromRaw($data['identifier']);
  180. $process = $processPool->getProcess($identifier);
  181. $process->bindConnection($decoder, $encoder);
  182. $fileChunk = $getFileChunk();
  183. if (0 === \count($fileChunk)) {
  184. $process->request(['action' => ParallelAction::RUNNER_THANK_YOU]);
  185. $processPool->endProcessIfKnown($identifier);
  186. return;
  187. }
  188. $process->request(['action' => ParallelAction::RUNNER_REQUEST_ANALYSIS, 'files' => $fileChunk]);
  189. });
  190. });
  191. $processesToSpawn = min(
  192. $this->parallelConfig->getMaxProcesses(),
  193. max(
  194. 1,
  195. (int) ceil($this->fileCount / $this->parallelConfig->getFilesPerProcess()),
  196. )
  197. );
  198. $processFactory = new ProcessFactory($this->input);
  199. for ($i = 0; $i < $processesToSpawn; ++$i) {
  200. $identifier = ProcessIdentifier::create();
  201. $process = $processFactory->create(
  202. $streamSelectLoop,
  203. new RunnerConfig(
  204. $this->isDryRun,
  205. $this->stopOnViolation,
  206. $this->parallelConfig,
  207. $this->configFile
  208. ),
  209. $identifier,
  210. $serverPort,
  211. );
  212. $processPool->addProcess($identifier, $process);
  213. $process->start(
  214. // [REACT] Handle workers' responses (multiple actions possible)
  215. function (array $workerResponse) use ($processPool, $process, $identifier, $getFileChunk, &$changed): void {
  216. // File analysis result (we want close-to-realtime progress with frequent cache savings)
  217. if (ParallelAction::WORKER_RESULT === $workerResponse['action']) {
  218. $fileAbsolutePath = $workerResponse['file'];
  219. $fileRelativePath = $this->directory->getRelativePathTo($fileAbsolutePath);
  220. // Dispatch an event for each file processed and dispatch its status (required for progress output)
  221. $this->dispatchEvent(
  222. FixerFileProcessedEvent::NAME,
  223. new FixerFileProcessedEvent($workerResponse['status'])
  224. );
  225. if (isset($workerResponse['fileHash'])) {
  226. $this->cacheManager->setFileHash($fileRelativePath, $workerResponse['fileHash']);
  227. }
  228. foreach ($workerResponse['errors'] ?? [] as $error) {
  229. $this->errorsManager->report(new Error(
  230. $error['type'],
  231. $error['filePath'],
  232. null !== $error['source']
  233. ? SourceExceptionFactory::fromArray($error['source'])
  234. : null,
  235. $error['appliedFixers'],
  236. $error['diff']
  237. ));
  238. }
  239. // Pass-back information about applied changes (only if there are any)
  240. if (isset($workerResponse['fixInfo'])) {
  241. $changed[$fileRelativePath] = $workerResponse['fixInfo'];
  242. if ($this->stopOnViolation) {
  243. $processPool->endAll();
  244. return;
  245. }
  246. }
  247. return;
  248. }
  249. if (ParallelAction::WORKER_GET_FILE_CHUNK === $workerResponse['action']) {
  250. // Request another chunk of files, if still available
  251. $fileChunk = $getFileChunk();
  252. if (0 === \count($fileChunk)) {
  253. $process->request(['action' => ParallelAction::RUNNER_THANK_YOU]);
  254. $processPool->endProcessIfKnown($identifier);
  255. return;
  256. }
  257. $process->request(['action' => ParallelAction::RUNNER_REQUEST_ANALYSIS, 'files' => $fileChunk]);
  258. return;
  259. }
  260. if (ParallelAction::WORKER_ERROR_REPORT === $workerResponse['action']) {
  261. throw WorkerException::fromRaw($workerResponse); // @phpstan-ignore-line
  262. }
  263. throw new ParallelisationException('Unsupported action: '.($workerResponse['action'] ?? 'n/a'));
  264. },
  265. // [REACT] Handle errors encountered during worker's execution
  266. static function (\Throwable $error) use ($processPool): void {
  267. $processPool->endAll();
  268. throw new ParallelisationException($error->getMessage(), $error->getCode(), $error);
  269. },
  270. // [REACT] Handle worker's shutdown
  271. static function ($exitCode, string $output) use ($processPool, $identifier): void {
  272. $processPool->endProcessIfKnown($identifier);
  273. if (0 === $exitCode || null === $exitCode) {
  274. return;
  275. }
  276. $errorsReported = Preg::matchAll(
  277. sprintf('/^(?:%s)([^\n]+)+/m', WorkerCommand::ERROR_PREFIX),
  278. $output,
  279. $matches
  280. );
  281. if ($errorsReported > 0) {
  282. throw WorkerException::fromRaw(json_decode($matches[1][0], true));
  283. }
  284. }
  285. );
  286. }
  287. $streamSelectLoop->run();
  288. return $changed;
  289. }
  290. /**
  291. * @return _RunResult
  292. */
  293. private function fixSequential(): array
  294. {
  295. $changed = [];
  296. $collection = $this->getLintingFileIterator();
  297. foreach ($collection as $file) {
  298. $fixInfo = $this->fixFile($file, $collection->currentLintingResult());
  299. // we do not need Tokens to still caching just fixed file - so clear the cache
  300. Tokens::clearCache();
  301. if (null !== $fixInfo) {
  302. $name = $this->directory->getRelativePathTo($file->__toString());
  303. $changed[$name] = $fixInfo;
  304. if ($this->stopOnViolation) {
  305. break;
  306. }
  307. }
  308. }
  309. return $changed;
  310. }
  311. /**
  312. * @return null|array{appliedFixers: list<string>, diff: string}
  313. */
  314. private function fixFile(\SplFileInfo $file, LintingResultInterface $lintingResult): ?array
  315. {
  316. $name = $file->getPathname();
  317. try {
  318. $lintingResult->check();
  319. } catch (LintingException $e) {
  320. $this->dispatchEvent(
  321. FixerFileProcessedEvent::NAME,
  322. new FixerFileProcessedEvent(FixerFileProcessedEvent::STATUS_INVALID)
  323. );
  324. $this->errorsManager->report(new Error(Error::TYPE_INVALID, $name, $e));
  325. return null;
  326. }
  327. $old = FileReader::createSingleton()->read($file->getRealPath());
  328. $tokens = Tokens::fromCode($old);
  329. $oldHash = $tokens->getCodeHash();
  330. $new = $old;
  331. $newHash = $oldHash;
  332. $appliedFixers = [];
  333. try {
  334. foreach ($this->fixers as $fixer) {
  335. // for custom fixers we don't know is it safe to run `->fix()` without checking `->supports()` and `->isCandidate()`,
  336. // thus we need to check it and conditionally skip fixing
  337. if (
  338. !$fixer instanceof AbstractFixer
  339. && (!$fixer->supports($file) || !$fixer->isCandidate($tokens))
  340. ) {
  341. continue;
  342. }
  343. $fixer->fix($file, $tokens);
  344. if ($tokens->isChanged()) {
  345. $tokens->clearEmptyTokens();
  346. $tokens->clearChanged();
  347. $appliedFixers[] = $fixer->getName();
  348. }
  349. }
  350. } catch (\ParseError $e) {
  351. $this->dispatchEvent(
  352. FixerFileProcessedEvent::NAME,
  353. new FixerFileProcessedEvent(FixerFileProcessedEvent::STATUS_LINT)
  354. );
  355. $this->errorsManager->report(new Error(Error::TYPE_LINT, $name, $e));
  356. return null;
  357. } catch (\Throwable $e) {
  358. $this->processException($name, $e);
  359. return null;
  360. }
  361. $fixInfo = null;
  362. if ([] !== $appliedFixers) {
  363. $new = $tokens->generateCode();
  364. $newHash = $tokens->getCodeHash();
  365. }
  366. // We need to check if content was changed and then applied changes.
  367. // But we can't simply check $appliedFixers, because one fixer may revert
  368. // work of other and both of them will mark collection as changed.
  369. // Therefore we need to check if code hashes changed.
  370. if ($oldHash !== $newHash) {
  371. $fixInfo = [
  372. 'appliedFixers' => $appliedFixers,
  373. 'diff' => $this->differ->diff($old, $new, $file),
  374. ];
  375. try {
  376. $this->linter->lintSource($new)->check();
  377. } catch (LintingException $e) {
  378. $this->dispatchEvent(
  379. FixerFileProcessedEvent::NAME,
  380. new FixerFileProcessedEvent(FixerFileProcessedEvent::STATUS_LINT)
  381. );
  382. $this->errorsManager->report(new Error(Error::TYPE_LINT, $name, $e, $fixInfo['appliedFixers'], $fixInfo['diff']));
  383. return null;
  384. }
  385. if (!$this->isDryRun) {
  386. $fileName = $file->getRealPath();
  387. if (!file_exists($fileName)) {
  388. throw new IOException(
  389. sprintf('Failed to write file "%s" (no longer) exists.', $file->getPathname()),
  390. 0,
  391. null,
  392. $file->getPathname()
  393. );
  394. }
  395. if (is_dir($fileName)) {
  396. throw new IOException(
  397. sprintf('Cannot write file "%s" as the location exists as directory.', $fileName),
  398. 0,
  399. null,
  400. $fileName
  401. );
  402. }
  403. if (!is_writable($fileName)) {
  404. throw new IOException(
  405. sprintf('Cannot write to file "%s" as it is not writable.', $fileName),
  406. 0,
  407. null,
  408. $fileName
  409. );
  410. }
  411. if (false === @file_put_contents($fileName, $new)) {
  412. $error = error_get_last();
  413. throw new IOException(
  414. sprintf('Failed to write file "%s", "%s".', $fileName, null !== $error ? $error['message'] : 'no reason available'),
  415. 0,
  416. null,
  417. $fileName
  418. );
  419. }
  420. }
  421. }
  422. $this->cacheManager->setFileHash($name, $newHash);
  423. $this->dispatchEvent(
  424. FixerFileProcessedEvent::NAME,
  425. new FixerFileProcessedEvent(null !== $fixInfo ? FixerFileProcessedEvent::STATUS_FIXED : FixerFileProcessedEvent::STATUS_NO_CHANGES, $name, $newHash)
  426. );
  427. return $fixInfo;
  428. }
  429. /**
  430. * Process an exception that occurred.
  431. */
  432. private function processException(string $name, \Throwable $e): void
  433. {
  434. $this->dispatchEvent(
  435. FixerFileProcessedEvent::NAME,
  436. new FixerFileProcessedEvent(FixerFileProcessedEvent::STATUS_EXCEPTION)
  437. );
  438. $this->errorsManager->report(new Error(Error::TYPE_EXCEPTION, $name, $e));
  439. }
  440. private function dispatchEvent(string $name, Event $event): void
  441. {
  442. if (null === $this->eventDispatcher) {
  443. return;
  444. }
  445. $this->eventDispatcher->dispatch($event, $name);
  446. }
  447. private function getLintingFileIterator(): LintingResultAwareFileIteratorInterface
  448. {
  449. $fileFilterIterator = $this->getFilteringFileIterator();
  450. return $this->linter->isAsync()
  451. ? new FileCachingLintingFileIterator($fileFilterIterator, $this->linter)
  452. : new LintingFileIterator($fileFilterIterator, $this->linter);
  453. }
  454. private function getFilteringFileIterator(): FileFilterIterator
  455. {
  456. if (null === $this->fileIterator) {
  457. throw new \RuntimeException('File iterator is not configured. Pass paths during Runner initialisation or set them after with `setFileIterator()`.');
  458. }
  459. return new FileFilterIterator(
  460. $this->fileIterator instanceof \IteratorAggregate
  461. ? $this->fileIterator->getIterator()
  462. : $this->fileIterator,
  463. $this->eventDispatcher,
  464. $this->cacheManager
  465. );
  466. }
  467. }